diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/manifest.txt b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..d95f12cd157321a5c540dad6b69eaa19ba9ec998 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-09-21T08:30:47+00:00 +host: sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-moonep +study_scripts_commit: d29f657 +campaign: moonep +backend: moonep +phase: tierbprobe +gbs: 8 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-MoE-Echo +megatron_branch_required: moonep_backend_async_grad_reduce_token_routing_analysis +megatron_branch_actual: moonep_backend_async_grad_reduce_token_routing_analysis +megatron_commit: a6a1a221 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/pip_freeze.txt b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d9fc4879c3ce5f88163350b93a0160e46cd890f --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.32.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@a6a1a2210babdf423efb969705f84584d06e97e0#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.2 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/provenance.txt b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/provenance.txt new file mode 100644 index 0000000000000000000000000000000000000000..8320f95e8aac257f2cfc8433a82b80db31d7a5ac --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/provenance.txt @@ -0,0 +1,90 @@ +============================================================== +PROVENANCE: moonep__tierbprobe__gbs8 +============================================================== +captured_at : 2026-09-21T08:30:33+00:00 +hostname : sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0 +kernel : 6.8.0-1043-nvidia +cwd : /workspace/blackwell_run_scripts +command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe --nsys + +---- GPU / driver -------------------------------------------- + 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + persistence/clocks: + 0, 120 MHz, 1965 MHz, 1000.00 W, 29 + 1, 1965 MHz, 1965 MHz, 1000.00 W, 35 + 2, 120 MHz, 1965 MHz, 1000.00 W, 34 + 3, 120 MHz, 1965 MHz, 1000.00 W, 30 + 4, 1965 MHz, 1965 MHz, 1000.00 W, 30 + 5, 120 MHz, 1965 MHz, 1000.00 W, 34 + 6, 120 MHz, 1965 MHz, 1000.00 W, 34 + 7, 120 MHz, 1965 MHz, 1000.00 W, 29 + + NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.32.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo moonep_backend_async_grad_reduce_token_routing_analysis a6a1a2210bab + Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-ddp-symmetric-params d8bc6667fad7 + blackwell_run_scripts granularity-moonep d29f6570cdce + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Sep 21 04:57 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Sep 21 04:54 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..79ea700a5d7c5e1c7286864b3db51d32502c7689 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv @@ -0,0 +1,192 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 4 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 3 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 977 +1, 979 +2, 999 +3, 975 +4, 1009 +5, 975 +6, 973 +7, 1001 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 13245 +1, 13243 +2, 13249 +3, 13257 +4, 13275 +5, 13263 +6, 13267 +7, 13259 +0, 14275 +1, 14313 +2, 14297 +3, 14297 +4, 14339 +5, 14363 +6, 14311 +7, 14335 +0, 15071 +1, 15077 +2, 15077 +3, 15079 +4, 15069 +5, 15079 +6, 15079 +7, 15069 +0, 15597 +1, 15603 +2, 15603 +3, 15605 +4, 15595 +5, 15605 +6, 15605 +7, 15595 +0, 15877 +1, 15883 +2, 15883 +3, 15885 +4, 15875 +5, 15883 +6, 15883 +7, 15875 +0, 26363 +1, 26355 +2, 26363 +3, 26383 +4, 26403 +5, 26391 +6, 26387 +7, 26403 +0, 26775 +1, 26781 +2, 26801 +3, 26783 +4, 26793 +5, 26781 +6, 26783 +7, 26793 +0, 27089 +1, 27067 +2, 27171 +3, 27179 +4, 27161 +5, 27141 +6, 27141 +7, 27143 +0, 27975 +1, 28011 +2, 28027 +3, 28039 +4, 28025 +5, 28037 +6, 28029 +7, 28041 +0, 35739 +1, 35739 +2, 35727 +3, 35745 +4, 35751 +5, 35743 +6, 35743 +7, 35751 +0, 35739 +1, 35739 +2, 35727 +3, 35745 +4, 35751 +5, 35743 +6, 35743 +7, 35751 +0, 35739 +1, 35739 +2, 6798 +3, 35099 +4, 6492 +5, 35375 +6, 35743 +7, 35709 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..5d2aa42505e136567bc282a533844a07b815d5fe --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env @@ -0,0 +1,105 @@ +backend=moonep +config=OLMOE_1B_7B_L16 +family=tierB +source_model=OLMoE-1B-7B-0924 +standard_num_layers=16 +base_rung= +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=a6a1a221 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=16 +hidden=2048 +ffn_hidden=1024 +moe_ffn_hidden=1024 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=6291456 +heads=16 +kv_groups=16 +kv_channels=128 +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +q_lora_rank= +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=0 +moe_layers=16 +rotary_base=10000 +rotary_percent=1.0 +rotary_scaling= +mscale= +mscale_all_dim= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +router_groups= +router_group_topk= +router_score_function=softmax +router_expert_bias=0 +router_bias_update_rate= +router_topk_scaling_factor= +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e7a106a244f18fde7772b974dc345f66f3e07a5c --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log @@ -0,0 +1,5033 @@ +[08:30:48] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + OLMOE_1B_7B_L16 / moonep (slots/rank=8 policy=matched) + E=64 K=8 H'=1024 m=8 + L=16 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ a6a1a221 +============================================================== +W0921 08:30:58.257000 669414 torch/distributed/run.py:766] +W0921 08:30:58.257000 669414 torch/distributed/run.py:766] ***************************************** +W0921 08:30:58.257000 669414 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 08:30:58.257000 669414 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 1024 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1024 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 16 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 3 is connected to 7 peer ranks. 2Expected number of connected peer ranks is : is connected to 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to [Gloo] Rank 7 peer ranks. Expected number of connected peer ranks is : 75 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 453 is connected to is connected to [Gloo] Rank is connected to 7777 peer ranks. peer ranks. is connected to peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 77 peer ranks. 7 + +Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to [Gloo] Rank 0 peer ranks. 0Expected number of connected peer ranks is : is connected to 00 peer ranks. +Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W921 08:31:15.459899416 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W921 08:31:15.459968080 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 08:31:15.459982690 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W921 08:31:15.462378192 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 08:31:15.462429602 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 08:31:15.462521910 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 08:31:15.462566564 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 08:31:15.623507430 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.742 seconds +time to initialize megatron (seconds): 7.352 +[after megatron is initialized] datetime: 2026-09-21 08:31:21 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 1281951744 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 7 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (46542848 elements, 46542848 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 3 (50737152 elements, 50737152 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 4 (50737152 elements, 50737152 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 5 (50737152 elements, 50737152 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight +Params for bucket 6 (50737152 elements, 50737152 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 7 (124131328 elements, 124131328 padded size): + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 2 (41943040 elements, 41943040 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 3 (41943040 elements, 41943040 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (41943040 elements, 41943040 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 5 (41943040 elements, 41943040 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 +Params for bucket 6 (41943040 elements, 41943040 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (41943040 elements, 41943040 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (41943040 elements, 41943040 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (41943040 elements, 41943040 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (41943040 elements, 41943040 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 +Params for bucket 11 (41943040 elements, 41943040 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 12 (41943040 elements, 41943040 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 13 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 15 (41943040 elements, 41943040 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 16 (41943040 elements, 41943040 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 +Params for bucket 17 (41943040 elements, 41943040 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 +Params for bucket 18 (41943040 elements, 41943040 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (41943040 elements, 41943040 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 20 (8388608 elements, 8388608 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 08:31:24 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 08:31:24 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (2738.36, 2739.42) + train/valid/test-data-iterators-setup ..........: (71.80, 90.87) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-09-21 08:31:24 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=2048, E=64, K=8, NvS=34800, NvS_padded=34816 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 136.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 139.26 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [16512, 5632, 1152, 2944, 1920, 3840, 1024, 384, 128, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[44, -1, -1, -1, -1, -1, -1, -1], [18, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [44, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [256, 4736, 8960, 256, 256, 768, 8704, 384, 9088, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[27, -1, -1, -1, -1, -1, -1, -1], [29, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [28, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [128, 384, 128, 1280, 128, 128, 1408, 384, 27904, 1664, 0, 0, 0, 0, 0, 0] | duplicated experts [[9, 8, -1, -1, -1, -1, -1, -1], [61, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [128, 7296, 4352, 1408, 8320, 4864, 128, 6528, 256, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[9, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [42, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [1, -1, -1, -1, -1, -1, -1, -1], [34, -1, -1, -1, -1, -1, -1, -1], [44, -1, -1, -1, -1, -1, -1, -1], [46, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-09-21 08:31:44] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 20069.8 | throughput per GPU (TFLOP/s/GPU): 1.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.118505E+01 | loss scale: 1.0 | grad norm: 35.488 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 6.71 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 6.92 +Number of parameters in most loaded shard in billions: 6.9171 +Theoretical memory footprints: weight and optimizer=49474.72 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 16627.5546875 | max allocated: 16999.91845703125 | reserved: 17484.0 | max reserved: 17484.0 + [2026-09-21 08:31:44] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 303.5 | throughput per GPU (TFLOP/s/GPU): 106.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.120947E+01 | loss scale: 1.0 | grad norm: 36.303 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=39230 dup_rows=20910 dup_rows_remote=18280 dedup_ratio=0.319061 router_tokens_per_expert=[2112, 840, 154, 442, 256, 546, 127, 105, 345, 246, 48, 2, 20, 356, 139, 2042, 40, 556, 1002, 1785, 122, 933, 937, 295, 1191, 128, 230, 1303, 428, 531, 83, 270, 220, 519, 674, 116, 586, 326, 475, 377, 518, 358, 428, 15, 1502, 769, 80, 545, 560, 724, 757, 57, 1857, 421, 259, 783, 521, 33, 627, 203, 63, 218, 431, 132] payload_rows_per_dst=[6018, 5050, 4768, 6034, 4874, 5174, 7312, 5396] dup_rows_per_dst=[2610, 1346, 1534, 2978, 1712, 1572, 6528, 2630] combine_rows_per_src=[6018, 5050, 4768, 6034, 4874, 5174, 7312, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45574 payload_rows_remote=40066 dup_rows=19962 dup_rows_remote=17962 dedup_ratio=0.304596 router_tokens_per_expert=[2682, 578, 147, 229, 346, 420, 91, 129, 424, 264, 65, 4, 73, 352, 299, 2021, 129, 1006, 1250, 1343, 139, 346, 503, 229, 720, 173, 187, 1436, 336, 591, 61, 250, 256, 466, 455, 56, 679, 186, 345, 387, 576, 481, 1129, 20, 1824, 853, 53, 416, 437, 897, 974, 33, 1917, 462, 85, 139, 767, 15, 722, 374, 19, 352, 458, 112] payload_rows_per_dst=[6558, 6870, 5464, 5508, 4358, 6596, 4670, 5550] dup_rows_per_dst=[2686, 3968, 1740, 2000, 1302, 4108, 1384, 2774] combine_rows_per_src=[6558, 6870, 5464, 5508, 4358, 6596, 4670, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45604 payload_rows_remote=40238 dup_rows=19932 dup_rows_remote=18352 dedup_ratio=0.304138 router_tokens_per_expert=[1527, 921, 105, 360, 112, 366, 25, 57, 143, 277, 79, 1, 24, 404, 90, 1583, 123, 345, 1412, 1637, 159, 940, 1194, 256, 933, 211, 138, 1574, 254, 494, 130, 217, 209, 821, 780, 158, 797, 316, 434, 512, 481, 643, 615, 32, 1658, 1014, 44, 441, 933, 711, 486, 104, 1414, 545, 206, 568, 632, 12, 1215, 262, 41, 204, 211, 178] payload_rows_per_dst=[5366, 5810, 7156, 5662, 6552, 5062, 5332, 4664] dup_rows_per_dst=[1580, 2220, 4660, 2240, 4818, 1478, 1774, 1162] combine_rows_per_src=[5366, 5810, 7156, 5662, 6552, 5062, 5332, 4664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46244 payload_rows_remote=39916 dup_rows=19292 dup_rows_remote=16484 dedup_ratio=0.294373 router_tokens_per_expert=[2092, 516, 168, 191, 207, 373, 140, 121, 261, 346, 45, 6, 83, 302, 234, 1282, 138, 774, 1349, 1784, 265, 707, 1082, 253, 756, 226, 248, 1349, 558, 413, 59, 261, 524, 744, 517, 118, 460, 279, 438, 564, 431, 638, 850, 20, 1490, 992, 84, 450, 343, 403, 577, 30, 2505, 443, 76, 472, 507, 18, 792, 200, 63, 504, 522, 125] payload_rows_per_dst=[5884, 6784, 6328, 5562, 5300, 6492, 4010, 5884] dup_rows_per_dst=[1732, 3344, 2808, 2178, 1988, 3418, 678, 3146] combine_rows_per_src=[5884, 6784, 6328, 5562, 5300, 6492, 4010, 5884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45314 payload_rows_remote=38784 dup_rows=20222 dup_rows_remote=16732 dedup_ratio=0.308563 router_tokens_per_expert=[2050, 812, 150, 305, 216, 492, 250, 81, 288, 425, 116, 4, 170, 284, 208, 1389, 113, 664, 800, 1481, 162, 409, 843, 259, 817, 141, 57, 990, 466, 562, 88, 431, 288, 597, 815, 158, 1317, 150, 454, 391, 629, 777, 856, 3, 1767, 427, 37, 514, 347, 1512, 497, 118, 1833, 318, 131, 228, 840, 29, 810, 357, 64, 378, 477, 126] payload_rows_per_dst=[6246, 4408, 4976, 5162, 5544, 6530, 6634, 5814] dup_rows_per_dst=[2466, 1360, 1524, 1942, 2796, 3490, 3334, 3310] combine_rows_per_src=[6246, 4408, 4976, 5162, 5544, 6530, 6634, 5814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45976 payload_rows_remote=40526 dup_rows=19560 dup_rows_remote=17414 dedup_ratio=0.298462 router_tokens_per_expert=[1966, 951, 59, 226, 169, 350, 87, 48, 433, 314, 63, 1, 29, 577, 218, 1905, 77, 517, 1296, 2014, 100, 735, 1557, 538, 1033, 54, 372, 1228, 649, 242, 124, 299, 154, 928, 656, 127, 628, 345, 526, 434, 265, 333, 414, 7, 1653, 626, 62, 840, 634, 502, 371, 35, 1685, 230, 200, 856, 437, 100, 581, 129, 7, 274, 273, 225] payload_rows_per_dst=[5800, 5664, 6458, 5542, 5450, 5850, 5714, 5498] dup_rows_per_dst=[1912, 2430, 3182, 2460, 2146, 2550, 2298, 2582] combine_rows_per_src=[5800, 5664, 6458, 5542, 5450, 5850, 5714, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45648 payload_rows_remote=38982 dup_rows=19888 dup_rows_remote=16632 dedup_ratio=0.303467 router_tokens_per_expert=[2304, 692, 196, 170, 306, 392, 32, 74, 248, 287, 24, 0, 73, 229, 170, 1720, 132, 1072, 1456, 1968, 75, 443, 816, 142, 975, 171, 90, 1365, 455, 971, 140, 322, 296, 476, 991, 76, 667, 191, 337, 498, 292, 744, 772, 37, 1712, 672, 55, 292, 271, 466, 787, 49, 2210, 583, 93, 115, 576, 4, 531, 200, 4, 386, 798, 77] payload_rows_per_dst=[6104, 6666, 6138, 6054, 6080, 4864, 3906, 5836] dup_rows_per_dst=[2228, 3256, 2134, 2924, 4034, 1238, 822, 3252] combine_rows_per_src=[6104, 6666, 6138, 6054, 6080, 4864, 3906, 5836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160686080 dispatch_meta_bytes=230040 combine_payload_bytes=160686080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160916120 ingress_bytes=185851904 link_bytes=185851904 token_bytes=321602200 weight_movement_bytes=25165824 total_nvlink_bytes=346768024 weight_to_token_ratio=0.078251 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.414118249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164110336 dispatch_meta_bytes=232112 combine_payload_bytes=164110336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=164342448 ingress_bytes=189276160 link_bytes=189276160 token_bytes=328452784 weight_movement_bytes=25165824 total_nvlink_bytes=353618608 weight_to_token_ratio=0.076619 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.440172881e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164814848 dispatch_meta_bytes=234360 combine_payload_bytes=164814848 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=165049208 ingress_bytes=189980672 link_bytes=189980672 token_bytes=329864056 weight_movement_bytes=25165824 total_nvlink_bytes=355029880 weight_to_token_ratio=0.076292 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.445533403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163495936 dispatch_meta_bytes=225600 combine_payload_bytes=163495936 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=163721536 ingress_bytes=264159232 link_bytes=264159232 token_bytes=327217472 weight_movement_bytes=100663296 total_nvlink_bytes=427880768 weight_to_token_ratio=0.307634 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.009946536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158859264 dispatch_meta_bytes=222064 combine_payload_bytes=158859264 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159081328 ingress_bytes=234356736 link_bytes=234356736 token_bytes=317940592 weight_movement_bytes=75497472 total_nvlink_bytes=393438064 weight_to_token_ratio=0.237458 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783183977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165994496 dispatch_meta_bytes=231760 combine_payload_bytes=165994496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166226256 ingress_bytes=191160320 link_bytes=191160320 token_bytes=332220752 weight_movement_bytes=25165824 total_nvlink_bytes=357386576 weight_to_token_ratio=0.075750 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=5888 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.454509162e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=222456 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159892728 ingress_bytes=184836096 link_bytes=184836096 token_bytes=319563000 weight_movement_bytes=25165824 total_nvlink_bytes=344728824 weight_to_token_ratio=0.078751 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406389124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50966 payload_rows_remote=42784 dup_rows=14570 dup_rows_remote=9052 dedup_ratio=0.222321 router_tokens_per_expert=[0, 2397, 58, 1, 6, 1, 141, 10, 1956, 0, 10, 70, 10, 2, 67, 1, 0, 0, 1, 0, 1, 4090, 0, 3966, 48, 76, 0, 1927, 602, 224, 258, 2013, 2, 0, 439, 409, 11, 0, 19, 2079, 5, 719, 1382, 0, 8, 2, 38, 2607, 1488, 1228, 0, 1435, 0, 27, 61, 113, 2, 371, 68, 6, 2283, 25, 3, 2] payload_rows_per_dst=[5012, 4108, 7934, 7138, 5220, 7098, 6274, 8182] dup_rows_per_dst=[216, 124, 2, 3158, 698, 2424, 2430, 5518] combine_rows_per_src=[5012, 4108, 7934, 7138, 5220, 7098, 6274, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49924 payload_rows_remote=42666 dup_rows=15612 dup_rows_remote=11910 dedup_ratio=0.238220 router_tokens_per_expert=[3, 1741, 99, 0, 0, 0, 1025, 0, 1172, 0, 330, 2, 4, 0, 55, 31, 7, 0, 0, 0, 0, 3474, 0, 3652, 29, 8, 0, 18, 659, 1089, 1, 668, 495, 11, 75, 19, 1, 0, 1, 4012, 18, 139, 1745, 9, 0, 1, 0, 3008, 119, 2812, 0, 531, 6, 0, 5, 2319, 2, 50, 1, 0, 3164, 158, 0, 0] payload_rows_per_dst=[4914, 4636, 7216, 7258, 8044, 3728, 7524, 6604] dup_rows_per_dst=[822, 2016, 3586, 3702, 1184, 96, 4060, 146] combine_rows_per_src=[4914, 4636, 7216, 7258, 8044, 3728, 7524, 6604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=41358 payload_rows_remote=33238 dup_rows=24178 dup_rows_remote=17978 dedup_ratio=0.368927 router_tokens_per_expert=[0, 3751, 141, 0, 0, 97, 1413, 1, 1692, 0, 734, 29, 70, 16, 175, 0, 0, 0, 1, 1, 1, 3940, 1, 3741, 1, 484, 0, 297, 191, 107, 98, 662, 34, 0, 2795, 1543, 0, 0, 0, 3472, 852, 105, 97, 183, 32, 0, 14, 1757, 0, 2890, 0, 55, 0, 6, 1, 979, 86, 2, 5, 6, 209, 1, 0, 0] payload_rows_per_dst=[8120, 8098, 7900, 3316, 8066, 3506, 1748, 604] dup_rows_per_dst=[6200, 4816, 694, 364, 11740, 312, 38, 14] combine_rows_per_src=[8120, 8098, 7900, 3316, 8066, 3506, 1748, 604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49634 payload_rows_remote=41666 dup_rows=15902 dup_rows_remote=10952 dedup_ratio=0.242645 router_tokens_per_expert=[8, 1197, 63, 0, 0, 0, 3272, 2, 3862, 7, 2319, 64, 13, 6, 6, 1, 0, 64, 2, 0, 0, 4026, 0, 4041, 0, 434, 0, 362, 106, 32, 28, 97, 72, 0, 83, 23, 0, 0, 0, 950, 5, 169, 3178, 6, 0, 0, 107, 2994, 124, 384, 0, 131, 1, 0, 42, 2756, 5, 2, 3, 1, 1633, 57, 24, 6] payload_rows_per_dst=[7228, 8000, 8100, 1922, 2216, 7968, 6072, 8128] dup_rows_per_dst=[1856, 4556, 114, 196, 40, 4950, 804, 3386] combine_rows_per_src=[7228, 8000, 8100, 1922, 2216, 7968, 6072, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51688 payload_rows_remote=44218 dup_rows=13848 dup_rows_remote=13844 dedup_ratio=0.211304 router_tokens_per_expert=[6, 2688, 121, 0, 0, 16, 155, 0, 2371, 0, 52, 35, 5, 1, 0, 0, 0, 1, 1, 0, 0, 3734, 1, 3584, 1, 36, 3, 1, 232, 2909, 724, 917, 12, 4, 56, 64, 24, 2, 7, 3943, 0, 769, 336, 0, 0, 0, 2, 1844, 2, 1898, 0, 886, 0, 4, 0, 2095, 1, 0, 15, 1, 2671, 153, 385, 0] payload_rows_per_dst=[6358, 7862, 7470, 7388, 7920, 2162, 6594, 5934] dup_rows_per_dst=[1954, 4234, 4, 3606, 304, 52, 3176, 518] combine_rows_per_src=[6358, 7862, 7470, 7388, 7920, 2162, 6594, 5934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45642 payload_rows_remote=43394 dup_rows=19894 dup_rows_remote=19870 dedup_ratio=0.303558 router_tokens_per_expert=[0, 598, 658, 0, 0, 0, 1484, 0, 227, 0, 29, 138, 425, 0, 1315, 0, 175, 1, 1, 0, 0, 1781, 1, 4044, 2, 1051, 0, 695, 1, 85, 7, 3725, 108, 0, 1, 11, 0, 0, 0, 1016, 23, 385, 928, 0, 0, 2, 29, 3206, 1803, 2523, 4, 80, 0, 20, 20, 15, 2093, 83, 2, 38, 3928, 0, 7, 0] payload_rows_per_dst=[4584, 3584, 8112, 7874, 2248, 4554, 6754, 7932] dup_rows_per_dst=[896, 684, 1308, 6800, 24, 1050, 2176, 6956] combine_rows_per_src=[4584, 3584, 8112, 7874, 2248, 4554, 6754, 7932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47958 payload_rows_remote=40862 dup_rows=17578 dup_rows_remote=14282 dedup_ratio=0.268219 router_tokens_per_expert=[3, 2846, 354, 0, 7, 11, 372, 0, 2591, 0, 8, 108, 307, 18, 31, 5, 5, 24, 4, 1, 3, 3882, 0, 2128, 133, 30, 0, 5, 339, 28, 176, 455, 12, 0, 2, 0, 0, 4, 339, 3943, 2, 3797, 2490, 5, 246, 0, 1279, 1269, 811, 517, 0, 792, 0, 0, 8, 2705, 63, 195, 7, 0, 338, 0, 70, 0] payload_rows_per_dst=[6816, 7096, 7792, 2026, 8056, 8006, 6834, 1332] dup_rows_per_dst=[2908, 3296, 46, 306, 544, 7782, 2682, 14] combine_rows_per_src=[6816, 7096, 7792, 2026, 8056, 8006, 6834, 1332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=175243264 dispatch_meta_bytes=207344 combine_payload_bytes=175243264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=175450608 ingress_bytes=200409088 link_bytes=200409088 token_bytes=350693872 weight_movement_bytes=25165824 total_nvlink_bytes=375859696 weight_to_token_ratio=0.071760 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.524881600e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=174759936 dispatch_meta_bytes=218304 combine_payload_bytes=174759936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=174978240 ingress_bytes=199925760 link_bytes=199925760 token_bytes=349738176 weight_movement_bytes=25165824 total_nvlink_bytes=374904000 weight_to_token_ratio=0.071956 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.521204032e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=136142848 dispatch_meta_bytes=204864 combine_payload_bytes=136142848 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=136347712 ingress_bytes=161308672 link_bytes=161308672 token_bytes=272490560 weight_movement_bytes=25165824 total_nvlink_bytes=297656384 weight_to_token_ratio=0.092355 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.227372612e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=170663936 dispatch_meta_bytes=210472 combine_payload_bytes=170663936 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=170874408 ingress_bytes=246161408 link_bytes=246161408 token_bytes=341538344 weight_movement_bytes=75497472 total_nvlink_bytes=417035816 weight_to_token_ratio=0.221051 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.873003891e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=181116928 dispatch_meta_bytes=232248 combine_payload_bytes=181116928 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=181349176 ingress_bytes=256614400 link_bytes=256614400 token_bytes=362466104 weight_movement_bytes=75497472 total_nvlink_bytes=437963576 weight_to_token_ratio=0.208288 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.952539082e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=177741824 dispatch_meta_bytes=253056 combine_payload_bytes=177741824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=177994880 ingress_bytes=202907648 link_bytes=202907648 token_bytes=355736704 weight_movement_bytes=25165824 total_nvlink_bytes=380902528 weight_to_token_ratio=0.070743 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.543892754e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=167370752 dispatch_meta_bytes=220576 combine_payload_bytes=167370752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=167591328 ingress_bytes=192536576 link_bytes=192536576 token_bytes=334962080 weight_movement_bytes=25165824 total_nvlink_bytes=360127904 weight_to_token_ratio=0.075130 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.464980880e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45070 payload_rows_remote=38428 dup_rows=20466 dup_rows_remote=15968 dedup_ratio=0.312286 router_tokens_per_expert=[2033, 819, 90, 195, 303, 380, 249, 104, 421, 205, 87, 1, 63, 329, 188, 1886, 190, 379, 928, 1688, 113, 871, 840, 513, 1247, 156, 95, 1178, 719, 479, 49, 221, 409, 651, 980, 135, 663, 200, 746, 507, 484, 222, 617, 16, 1512, 711, 29, 747, 727, 598, 846, 70, 1445, 570, 266, 267, 506, 23, 668, 211, 78, 309, 349, 187] payload_rows_per_dst=[6050, 4942, 5030, 6094, 5590, 5202, 6642, 5520] dup_rows_per_dst=[2296, 1418, 1636, 3196, 2992, 1912, 4498, 2518] combine_rows_per_src=[6050, 4942, 5030, 6094, 5590, 5202, 6642, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54256 payload_rows_remote=46512 dup_rows=11280 dup_rows_remote=11262 dedup_ratio=0.172119 router_tokens_per_expert=[356, 0, 0, 0, 0, 1, 0, 0, 0, 2, 49, 743, 0, 0, 1, 0, 2, 4090, 1728, 7, 0, 4, 0, 0, 0, 0, 0, 24, 1, 11, 0, 1, 3844, 6, 27, 13, 7, 0, 3863, 0, 0, 0, 48, 56, 3712, 1, 23, 48, 1191, 862, 0, 0, 1056, 0, 0, 3759, 3040, 0, 0, 0, 4044, 8, 9, 131] payload_rows_per_dst=[7544, 3334, 8188, 7744, 7814, 7566, 3930, 8136] dup_rows_per_dst=[688, 368, 3474, 18, 18, 210, 176, 6328] combine_rows_per_src=[7544, 3334, 8188, 7744, 7814, 7566, 3930, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48416 payload_rows_remote=40226 dup_rows=17120 dup_rows_remote=9098 dedup_ratio=0.261230 router_tokens_per_expert=[1, 101, 0, 1, 0, 0, 1, 0, 47, 2585, 0, 2704, 31, 0, 2, 0, 0, 1456, 5, 16, 0, 0, 0, 19, 32, 2, 6, 73, 100, 2715, 13, 0, 1630, 263, 1, 12, 0, 0, 31, 1, 0, 93, 0, 1614, 3691, 0, 3100, 249, 145, 1041, 781, 0, 1767, 0, 0, 4024, 4021, 0, 0, 0, 1, 36, 357, 0] payload_rows_per_dst=[8050, 7114, 2950, 5660, 3788, 7120, 5544, 8190] dup_rows_per_dst=[206, 3624, 42, 222, 26, 2992, 1986, 8022] combine_rows_per_src=[8050, 7114, 2950, 5660, 3788, 7120, 5544, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49320 payload_rows_remote=41226 dup_rows=16216 dup_rows_remote=16128 dedup_ratio=0.247437 router_tokens_per_expert=[13, 0, 0, 0, 0, 0, 21, 0, 15, 172, 17, 30, 0, 0, 0, 0, 467, 3847, 775, 29, 3, 0, 0, 0, 0, 0, 0, 2375, 18, 4, 0, 274, 3810, 3, 3181, 221, 455, 0, 3807, 0, 0, 2, 0, 3, 564, 0, 0, 0, 4052, 61, 13, 3, 226, 0, 258, 3942, 3140, 4, 0, 0, 883, 27, 49, 4] payload_rows_per_dst=[8094, 890, 8048, 8002, 8054, 1138, 8092, 7002] dup_rows_per_dst=[88, 30, 4468, 4960, 5006, 0, 452, 1212] combine_rows_per_src=[8094, 890, 8048, 8002, 8054, 1138, 8092, 7002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50406 payload_rows_remote=44088 dup_rows=15130 dup_rows_remote=13304 dedup_ratio=0.230865 router_tokens_per_expert=[35, 4, 0, 0, 0, 0, 0, 1, 0, 2073, 5, 244, 78, 5, 4, 0, 4, 1137, 82, 55, 0, 1, 0, 0, 9, 1, 19, 0, 2317, 0, 0, 9, 3688, 81, 20, 11, 2388, 0, 1443, 0, 0, 0, 0, 0, 3959, 484, 0, 28, 3999, 122, 0, 0, 3538, 6, 0, 3934, 168, 0, 3, 4, 984, 1825, 0, 0] payload_rows_per_dst=[7924, 5928, 2526, 4638, 8068, 6318, 8186, 6818] dup_rows_per_dst=[24, 556, 32, 72, 4308, 1826, 6462, 1850] combine_rows_per_src=[7924, 5928, 2526, 4638, 8068, 6318, 8186, 6818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=41866 payload_rows_remote=34560 dup_rows=23670 dup_rows_remote=21770 dedup_ratio=0.361176 router_tokens_per_expert=[391, 24, 0, 0, 4, 0, 0, 6, 0, 0, 9, 4058, 0, 0, 0, 1, 2, 2723, 1840, 0, 31, 0, 0, 7, 0, 0, 0, 4, 0, 7, 0, 0, 2688, 198, 64, 11, 0, 2, 70, 0, 0, 0, 0, 3263, 3826, 0, 0, 0, 4072, 3559, 0, 0, 4068, 0, 0, 1660, 0, 0, 0, 0, 164, 7, 9, 0] payload_rows_per_dst=[3934, 8156, 7306, 5382, 514, 8044, 8180, 350] dup_rows_per_dst=[236, 8116, 1900, 16, 176, 6134, 7082, 10] combine_rows_per_src=[3934, 8156, 7306, 5382, 514, 8044, 8180, 350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49028 payload_rows_remote=43316 dup_rows=16508 dup_rows_remote=15028 dedup_ratio=0.251892 router_tokens_per_expert=[5, 0, 0, 1811, 7, 0, 2051, 0, 0, 1382, 1895, 0, 0, 0, 9, 267, 7, 1340, 0, 2, 15, 0, 0, 0, 0, 0, 5, 258, 938, 4073, 0, 0, 1441, 197, 27, 16, 0, 493, 2961, 0, 0, 0, 0, 18, 151, 538, 3212, 1, 3, 50, 3764, 0, 59, 0, 0, 3590, 219, 0, 0, 0, 13, 0, 27, 1923] payload_rows_per_dst=[8028, 5746, 2684, 8188, 5712, 6806, 7542, 4322] dup_rows_per_dst=[6900, 1478, 44, 3146, 1480, 3326, 92, 42] combine_rows_per_src=[8028, 5746, 2684, 8188, 5712, 6806, 7542, 4322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46326 payload_rows_remote=38134 dup_rows=19210 dup_rows_remote=12432 dedup_ratio=0.293121 router_tokens_per_expert=[171, 0, 0, 0, 2404, 13, 0, 67, 505, 3762, 0, 0, 1, 0, 0, 0, 376, 3728, 692, 0, 12, 1, 0, 0, 79, 0, 0, 0, 0, 0, 188, 18, 3711, 0, 64, 47, 2, 2230, 3879, 0, 0, 0, 0, 0, 4032, 2772, 0, 1, 132, 219, 233, 0, 3217, 1, 24, 171, 13, 0, 1, 0, 0, 0, 2, 0] payload_rows_per_dst=[5514, 8192, 7962, 7522, 7880, 8100, 1128, 28] dup_rows_per_dst=[138, 6778, 1656, 470, 4564, 5510, 90, 4] combine_rows_per_src=[5514, 8192, 7962, 7522, 7880, 8100, 1128, 28] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=190513152 dispatch_meta_bytes=231096 combine_payload_bytes=190513152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=190744248 ingress_bytes=215678976 link_bytes=215678976 token_bytes=381257400 weight_movement_bytes=25165824 total_nvlink_bytes=406423224 weight_to_token_ratio=0.066007 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.641067804e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157401088 dispatch_meta_bytes=217584 combine_payload_bytes=157401088 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=157618672 ingress_bytes=207732736 link_bytes=207732736 token_bytes=315019760 weight_movement_bytes=50331648 total_nvlink_bytes=365351408 weight_to_token_ratio=0.159773 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.580606099e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=164765696 dispatch_meta_bytes=197296 combine_payload_bytes=164765696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=164962992 ingress_bytes=189931520 link_bytes=189931520 token_bytes=329728688 weight_movement_bytes=25165824 total_nvlink_bytes=354894512 weight_to_token_ratio=0.076323 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.445159413e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=168861696 dispatch_meta_bytes=229416 combine_payload_bytes=168861696 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=169091112 ingress_bytes=219193344 link_bytes=219193344 token_bytes=337952808 weight_movement_bytes=50331648 total_nvlink_bytes=388284456 weight_to_token_ratio=0.148931 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=25216 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.667808084e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=180584448 dispatch_meta_bytes=229568 combine_payload_bytes=180584448 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=180814016 ingress_bytes=230916096 link_bytes=230916096 token_bytes=361398464 weight_movement_bytes=50331648 total_nvlink_bytes=411730112 weight_to_token_ratio=0.139269 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.757004682e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=141557760 dispatch_meta_bytes=225320 combine_payload_bytes=141557760 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=141783080 ingress_bytes=166723584 link_bytes=166723584 token_bytes=283340840 weight_movement_bytes=25165824 total_nvlink_bytes=308506664 weight_to_token_ratio=0.088818 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.268573836e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156196864 dispatch_meta_bytes=202264 combine_payload_bytes=156196864 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=156399128 ingress_bytes=181362688 link_bytes=181362688 token_bytes=312595992 weight_movement_bytes=25165824 total_nvlink_bytes=337761816 weight_to_token_ratio=0.080506 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.379960503e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=177422336 dispatch_meta_bytes=233376 combine_payload_bytes=177422336 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=177655712 ingress_bytes=278085632 link_bytes=278085632 token_bytes=355078048 weight_movement_bytes=100663296 total_nvlink_bytes=455741344 weight_to_token_ratio=0.283496 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.115910349e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54644 payload_rows_remote=47510 dup_rows=10892 dup_rows_remote=10890 dedup_ratio=0.166199 router_tokens_per_expert=[0, 0, 0, 0, 3284, 4068, 0, 3750, 0, 5, 0, 0, 12, 0, 4, 2, 832, 0, 0, 0, 224, 9, 0, 0, 0, 0, 0, 3556, 0, 1, 11, 0, 4012, 4096, 2445, 4081, 0, 0, 0, 160, 0, 0, 0, 0, 2, 0, 26, 0, 0, 0, 0, 0, 2038, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 140] payload_rows_per_dst=[6568, 38, 8178, 7134, 8190, 8186, 8192, 8158] dup_rows_per_dst=[0, 8, 1452, 2, 5044, 32, 4090, 264] combine_rows_per_src=[6568, 38, 8178, 7134, 8190, 8186, 8192, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53800 payload_rows_remote=46256 dup_rows=11736 dup_rows_remote=11736 dedup_ratio=0.179077 router_tokens_per_expert=[0, 0, 0, 9, 2333, 3769, 0, 3889, 9, 606, 111, 1, 0, 22, 0, 1, 103, 189, 53, 0, 0, 1, 0, 0, 1, 0, 116, 3579, 0, 3536, 0, 0, 23, 3934, 3499, 4082, 0, 801, 0, 375, 2, 0, 0, 0, 0, 6, 576, 0, 0, 0, 0, 564, 506, 0, 0, 69, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[4684, 1436, 8092, 7508, 8174, 8176, 8186, 7544] dup_rows_per_dst=[0, 64, 378, 1716, 6462, 1156, 1960, 0] combine_rows_per_src=[4684, 1436, 8092, 7508, 8174, 8176, 8186, 7544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48318 payload_rows_remote=40130 dup_rows=17218 dup_rows_remote=4612 dedup_ratio=0.262726 router_tokens_per_expert=[0, 0, 4, 0, 0, 3697, 0, 3941, 0, 4003, 0, 0, 0, 14, 1, 142, 1, 0, 25, 0, 1, 33, 0, 8, 0, 2, 130, 4051, 0, 290, 0, 98, 4, 3990, 0, 4048, 0, 0, 0, 0, 0, 6, 2418, 3919, 6, 0, 0, 0, 37, 0, 16, 0, 378, 2, 2, 133, 1, 0, 0, 1003, 81, 7, 0, 276] payload_rows_per_dst=[8, 8042, 7984, 8184, 8, 8188, 8148, 7756] dup_rows_per_dst=[0, 278, 34, 958, 0, 12606, 968, 2374] combine_rows_per_src=[8, 8042, 7984, 8184, 8, 8188, 8148, 7756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43936 payload_rows_remote=35750 dup_rows=21600 dup_rows_remote=5002 dedup_ratio=0.329590 router_tokens_per_expert=[0, 0, 0, 0, 4090, 3802, 0, 4092, 0, 3827, 0, 0, 4, 0, 0, 1854, 433, 1, 145, 6, 4, 0, 2, 0, 10, 0, 0, 2069, 0, 0, 11, 0, 352, 3647, 0, 4049, 0, 1, 0, 6, 56, 0, 36, 1, 0, 0, 0, 4, 0, 0, 0, 273, 3904, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 2] payload_rows_per_dst=[8186, 8090, 4234, 42, 8132, 6906, 8168, 178] dup_rows_per_dst=[16598, 3280, 270, 0, 1194, 72, 186, 0] combine_rows_per_src=[8186, 8090, 4234, 42, 8132, 6906, 8168, 178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=42812 payload_rows_remote=34710 dup_rows=22724 dup_rows_remote=17974 dedup_ratio=0.346741 router_tokens_per_expert=[1, 0, 0, 0, 11, 6, 0, 4077, 0, 132, 4, 58, 0, 47, 1, 3906, 0, 1393, 494, 94, 0, 119, 0, 3, 0, 1, 0, 2961, 6, 3918, 2148, 12, 0, 2886, 451, 85, 12, 0, 0, 2517, 68, 1, 772, 0, 22, 0, 1, 0, 276, 2, 0, 0, 0, 4, 16, 0, 0, 5, 53, 898, 2541, 1, 0, 2765] payload_rows_per_dst=[1392, 8102, 8190, 8118, 6840, 1772, 582, 7816] dup_rows_per_dst=[10, 4750, 4170, 4052, 4892, 126, 14, 4710] combine_rows_per_src=[1392, 8102, 8190, 8118, 6840, 1772, 582, 7816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52872 payload_rows_remote=44840 dup_rows=12664 dup_rows_remote=12258 dedup_ratio=0.193237 router_tokens_per_expert=[0, 0, 0, 0, 3350, 3971, 0, 3981, 0, 2313, 0, 13, 6, 0, 0, 3436, 0, 234, 2, 0, 2, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 2, 0, 3832, 4077, 416, 754, 0, 0, 0, 143, 58, 0, 144, 3, 0, 0, 310, 82, 32, 0, 0, 23, 1313, 0, 151, 0, 0, 0, 0, 157, 1, 0, 0, 1] payload_rows_per_dst=[7396, 7948, 8032, 6378, 8146, 2402, 7592, 4978] dup_rows_per_dst=[2586, 5136, 406, 0, 3572, 300, 664, 0] combine_rows_per_src=[7396, 7948, 8032, 6378, 8146, 2402, 7592, 4978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=194600960 dispatch_meta_bytes=233600 combine_payload_bytes=194600960 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=194834560 ingress_bytes=270098432 link_bytes=270098432 token_bytes=389435520 weight_movement_bytes=75497472 total_nvlink_bytes=464932992 weight_to_token_ratio=0.193864 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.055136986e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=189464576 dispatch_meta_bytes=231968 combine_payload_bytes=189464576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189696544 ingress_bytes=214630400 link_bytes=214630400 token_bytes=379161120 weight_movement_bytes=25165824 total_nvlink_bytes=404326944 weight_to_token_ratio=0.066372 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.633089352e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52332 payload_rows_remote=51772 dup_rows=13204 dup_rows_remote=13196 dedup_ratio=0.201477 router_tokens_per_expert=[0, 0, 2654, 0, 2711, 4079, 2, 4093, 0, 0, 1, 0, 16, 0, 0, 1776, 0, 97, 16, 134, 0, 0, 1, 20, 0, 0, 144, 2518, 0, 3949, 4, 0, 0, 3983, 16, 4091, 0, 261, 0, 7, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 20, 1, 0, 0, 0, 0, 0, 0, 1415, 0, 0, 0, 756] payload_rows_per_dst=[7604, 3554, 8192, 8038, 560, 8188, 8008, 8188] dup_rows_per_dst=[3130, 32, 530, 5192, 8, 0, 0, 4312] combine_rows_per_src=[7604, 3554, 8192, 8038, 560, 8188, 8008, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48648 payload_rows_remote=42678 dup_rows=16888 dup_rows_remote=14488 dedup_ratio=0.257690 router_tokens_per_expert=[61, 1534, 1597, 0, 0, 0, 634, 0, 503, 18, 2, 5, 18, 0, 667, 0, 4, 0, 0, 0, 0, 3023, 0, 2625, 0, 2, 1, 1654, 10, 1168, 20, 2386, 287, 0, 602, 25, 0, 2, 6, 3694, 263, 40, 2850, 2, 0, 0, 926, 1843, 353, 1518, 0, 697, 5, 0, 0, 1612, 994, 164, 58, 0, 884, 0, 11, 0] payload_rows_per_dst=[5916, 2242, 5258, 7182, 7716, 7518, 5970, 6846] dup_rows_per_dst=[1736, 184, 0, 3300, 1516, 4330, 2400, 3422] combine_rows_per_src=[5916, 2242, 5258, 7182, 7716, 7518, 5970, 6846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=164372480 dispatch_meta_bytes=178968 combine_payload_bytes=164372480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=164551448 ingress_bytes=189538304 link_bytes=189538304 token_bytes=328923928 weight_movement_bytes=25165824 total_nvlink_bytes=354089752 weight_to_token_ratio=0.076510 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.442167494e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=146432000 dispatch_meta_bytes=163008 combine_payload_bytes=146432000 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=146595008 ingress_bytes=221929472 link_bytes=221929472 token_bytes=293027008 weight_movement_bytes=75497472 total_nvlink_bytes=368524480 weight_to_token_ratio=0.257647 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.688626857e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=142172160 dispatch_meta_bytes=210736 combine_payload_bytes=142172160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=142382896 ingress_bytes=167337984 link_bytes=167337984 token_bytes=284555056 weight_movement_bytes=25165824 total_nvlink_bytes=309720880 weight_to_token_ratio=0.088439 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.273248710e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=183664640 dispatch_meta_bytes=228392 combine_payload_bytes=183664640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183893032 ingress_bytes=208830464 link_bytes=208830464 token_bytes=367557672 weight_movement_bytes=25165824 total_nvlink_bytes=392723496 weight_to_token_ratio=0.068468 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588958541e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=212058112 dispatch_meta_bytes=259872 combine_payload_bytes=212058112 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=212317984 ingress_bytes=287555584 link_bytes=287555584 token_bytes=424376096 weight_movement_bytes=75497472 total_nvlink_bytes=499873568 weight_to_token_ratio=0.177902 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=9600 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.187965742e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=174809088 dispatch_meta_bytes=228664 combine_payload_bytes=174809088 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=175037752 ingress_bytes=250306560 link_bytes=250306560 token_bytes=349846840 weight_movement_bytes=75497472 total_nvlink_bytes=425344312 weight_to_token_ratio=0.215801 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.904543708e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50530 payload_rows_remote=50484 dup_rows=15006 dup_rows_remote=15006 dedup_ratio=0.228973 router_tokens_per_expert=[18, 8, 4092, 168, 0, 0, 0, 0, 1396, 0, 1852, 14, 0, 0, 0, 5, 7, 676, 4, 4096, 0, 3422, 1545, 0, 0, 0, 1, 3666, 0, 0, 14, 8, 0, 3, 0, 81, 0, 0, 0, 1, 3979, 5, 0, 37, 0, 0, 6, 1116, 113, 16, 0, 2316, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 6, 2, 0] payload_rows_per_dst=[8190, 8076, 6326, 46, 8192, 8180, 6114, 5406] dup_rows_per_dst=[382, 5790, 1312, 0, 170, 4904, 2446, 2] combine_rows_per_src=[8190, 8076, 6326, 46, 8192, 8180, 6114, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43504 payload_rows_remote=35470 dup_rows=22032 dup_rows_remote=14440 dedup_ratio=0.336182 router_tokens_per_expert=[13, 0, 0, 9, 32, 46, 0, 0, 87, 55, 0, 0, 0, 0, 0, 64, 1, 182, 1867, 3969, 2175, 0, 2846, 0, 0, 2, 72, 2697, 0, 43, 12, 2947, 0, 0, 266, 0, 6, 0, 0, 0, 762, 0, 34, 721, 10, 0, 3957, 46, 750, 232, 85, 917, 17, 0, 33, 0, 455, 0, 3731, 0, 0, 3626, 1, 0] payload_rows_per_dst=[200, 410, 4452, 6468, 8190, 8086, 7664, 8034] dup_rows_per_dst=[0, 2, 264, 8812, 292, 2974, 2096, 7592] combine_rows_per_src=[200, 410, 4452, 6468, 8190, 8086, 7664, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47536 payload_rows_remote=39758 dup_rows=18000 dup_rows_remote=14004 dedup_ratio=0.274658 router_tokens_per_expert=[0, 4, 3696, 0, 3984, 62, 0, 632, 3675, 205, 12, 0, 0, 14, 0, 0, 442, 0, 36, 4040, 0, 18, 3456, 0, 0, 0, 283, 1981, 0, 0, 10, 3514, 0, 0, 13, 0, 0, 1864, 3, 0, 70, 0, 351, 0, 1, 0, 0, 272, 0, 553, 0, 36, 0, 59, 278, 0, 0, 0, 0, 99, 0, 3105, 0, 0] payload_rows_per_dst=[8192, 7778, 2686, 7194, 6058, 1386, 8032, 6210] dup_rows_per_dst=[8564, 3996, 840, 420, 3248, 2, 732, 198] combine_rows_per_src=[8192, 7778, 2686, 7194, 6058, 1386, 8032, 6210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=35168 payload_rows_remote=26982 dup_rows=30368 dup_rows_remote=16398 dedup_ratio=0.463379 router_tokens_per_expert=[0, 3754, 4062, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 39, 0, 0, 0, 1, 2932, 4091, 956, 3775, 707, 0, 0, 0, 4, 3979, 0, 0, 0, 173, 0, 0, 1, 9, 3252, 0, 45, 0, 4, 0, 0, 76, 0, 0, 32, 6, 240, 20, 1, 271, 7, 0, 0, 0, 0, 0, 4083, 0, 0, 239, 0, 0] payload_rows_per_dst=[8186, 8020, 8192, 350, 6584, 1000, 2358, 478] dup_rows_per_dst=[13970, 28, 16084, 4, 30, 118, 134, 0] combine_rows_per_src=[8186, 8020, 8192, 350, 6584, 1000, 2358, 478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53364 payload_rows_remote=45412 dup_rows=12172 dup_rows_remote=9988 dedup_ratio=0.185730 router_tokens_per_expert=[8, 3, 7, 0, 0, 805, 0, 0, 3023, 0, 0, 127, 0, 0, 0, 0, 1, 1215, 326, 4092, 7, 3369, 2739, 150, 0, 0, 2, 3905, 0, 0, 0, 2402, 0, 0, 24, 12, 0, 0, 9, 0, 1004, 0, 0, 0, 0, 0, 14, 94, 0, 112, 1, 1297, 330, 0, 0, 2, 0, 0, 3702, 1, 0, 3985, 0, 0] payload_rows_per_dst=[1636, 8178, 7952, 4808, 8192, 7520, 7106, 7972] dup_rows_per_dst=[10, 5932, 2184, 0, 82, 2108, 1856, 0] combine_rows_per_src=[1636, 8178, 7952, 4808, 8192, 7520, 7106, 7972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52818 payload_rows_remote=45906 dup_rows=12718 dup_rows_remote=12708 dedup_ratio=0.194061 router_tokens_per_expert=[347, 0, 3565, 7, 0, 12, 23, 46, 2076, 0, 0, 129, 0, 0, 0, 0, 0, 985, 169, 3256, 0, 218, 4058, 0, 0, 0, 0, 1126, 0, 0, 0, 4015, 37, 1, 0, 0, 262, 130, 0, 4, 0, 3, 3, 21, 0, 0, 3434, 0, 0, 6, 1, 287, 742, 1, 10, 0, 0, 0, 3766, 0, 0, 3957, 0, 71] payload_rows_per_dst=[7676, 4396, 2236, 8030, 7236, 6912, 8140, 8192] dup_rows_per_dst=[324, 14, 72, 2252, 144, 10, 2506, 7396] combine_rows_per_src=[7676, 4396, 2236, 8030, 7236, 6912, 8140, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=206782464 dispatch_meta_bytes=261960 combine_payload_bytes=206782464 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=207044424 ingress_bytes=257114112 link_bytes=257114112 token_bytes=413826888 weight_movement_bytes=50331648 total_nvlink_bytes=464158536 weight_to_token_ratio=0.121625 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18304 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.956341313e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=145285120 dispatch_meta_bytes=199640 combine_payload_bytes=145285120 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=145484760 ingress_bytes=220782592 link_bytes=220782592 token_bytes=290769880 weight_movement_bytes=75497472 total_nvlink_bytes=366267352 weight_to_token_ratio=0.259647 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.679900425e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60294 payload_rows_remote=52102 dup_rows=5242 dup_rows_remote=4508 dedup_ratio=0.079987 router_tokens_per_expert=[0, 0, 35, 4020, 5, 4, 1, 0, 5, 0, 16, 0, 0, 103, 0, 148, 0, 608, 3864, 4092, 165, 4, 3076, 0, 0, 0, 52, 3486, 0, 0, 0, 4000, 0, 0, 0, 361, 0, 0, 10, 0, 4, 0, 81, 0, 211, 0, 3544, 6, 762, 2, 29, 0, 0, 3, 0, 0, 0, 0, 4070, 0, 0, 1, 0, 0] payload_rows_per_dst=[8078, 4632, 7916, 8076, 8192, 7652, 7608, 8140] dup_rows_per_dst=[52, 112, 1358, 2800, 734, 40, 144, 2] combine_rows_per_src=[8078, 4632, 7916, 8076, 8192, 7652, 7608, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=162848768 dispatch_meta_bytes=215048 combine_payload_bytes=162848768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=163063816 ingress_bytes=188014592 link_bytes=188014592 token_bytes=325912584 weight_movement_bytes=25165824 total_nvlink_bytes=351078408 weight_to_token_ratio=0.077216 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=15744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.430573806e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=110518272 dispatch_meta_bytes=173520 combine_payload_bytes=110518272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=110691792 ingress_bytes=135684096 link_bytes=135684096 token_bytes=221210064 weight_movement_bytes=25165824 total_nvlink_bytes=246375888 weight_to_token_ratio=0.113764 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.032399196e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=186007552 dispatch_meta_bytes=221600 combine_payload_bytes=186007552 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=186229152 ingress_bytes=311836672 link_bytes=311836672 token_bytes=372236704 weight_movement_bytes=125829120 total_nvlink_bytes=498065824 weight_to_token_ratio=0.338035 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=10368 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.372716767e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=188030976 dispatch_meta_bytes=234456 combine_payload_bytes=188030976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188265432 ingress_bytes=213196800 link_bytes=213196800 token_bytes=376296408 weight_movement_bytes=25165824 total_nvlink_bytes=401462232 weight_to_token_ratio=0.066878 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=11136 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.622181313e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49858 payload_rows_remote=44182 dup_rows=15678 dup_rows_remote=14760 dedup_ratio=0.239227 router_tokens_per_expert=[10, 0, 8, 3, 0, 2, 0, 0, 744, 0, 2, 1794, 0, 0, 69, 4, 485, 2575, 4063, 0, 0, 5, 0, 0, 0, 0, 0, 1303, 764, 2681, 0, 0, 3788, 3, 237, 0, 33, 3, 1099, 0, 0, 0, 0, 66, 2462, 0, 153, 4, 1, 1610, 0, 0, 587, 0, 0, 4061, 4091, 0, 0, 0, 0, 2, 31, 25] payload_rows_per_dst=[8148, 5106, 8190, 6450, 7674, 424, 5676, 8190] dup_rows_per_dst=[20, 120, 6066, 3046, 454, 22, 918, 5032] combine_rows_per_src=[8148, 5106, 8190, 6450, 7674, 424, 5676, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=213409792 dispatch_meta_bytes=226440 combine_payload_bytes=213409792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213636232 ingress_bytes=238575616 link_bytes=238575616 token_bytes=427046024 weight_movement_bytes=25165824 total_nvlink_bytes=452211848 weight_to_token_ratio=0.058930 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=26496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.815284779e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49674 payload_rows_remote=41534 dup_rows=15862 dup_rows_remote=3806 dedup_ratio=0.242035 router_tokens_per_expert=[0, 3, 0, 3914, 1, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 1395, 0, 0, 0, 0, 3357, 1039, 0, 3479, 2895, 0, 4, 0, 0, 0, 408, 4091, 79, 0, 0, 0, 0, 2, 10, 1343, 1, 0, 0, 4, 0, 551, 30, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 2005] payload_rows_per_dst=[8044, 7828, 8184, 8140, 8036, 3996, 1154, 4292] dup_rows_per_dst=[2, 0, 2250, 12056, 394, 794, 16, 350] combine_rows_per_src=[8044, 7828, 8184, 8140, 8036, 3996, 1154, 4292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46378 payload_rows_remote=38186 dup_rows=19158 dup_rows_remote=10506 dedup_ratio=0.292328 router_tokens_per_expert=[0, 44, 0, 4064, 0, 0, 2776, 0, 0, 13, 0, 0, 0, 0, 11, 0, 1, 0, 3, 0, 0, 36, 258, 0, 1435, 3796, 4081, 1, 105, 0, 0, 0, 0, 58, 0, 281, 3782, 131, 0, 61, 0, 8, 1, 0, 161, 0, 0, 0, 3945, 0, 1691, 0, 0, 5, 0, 1543, 0, 256, 57, 118, 0, 0, 0, 4046] payload_rows_per_dst=[3744, 8162, 552, 3080, 8012, 8178, 6458, 8192] dup_rows_per_dst=[26, 1884, 44, 2, 614, 7916, 20, 8652] combine_rows_per_src=[3744, 8162, 552, 3080, 8012, 8178, 6458, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47664 payload_rows_remote=39512 dup_rows=17872 dup_rows_remote=17232 dedup_ratio=0.272705 router_tokens_per_expert=[0, 33, 3778, 3867, 0, 0, 4093, 0, 9, 3, 0, 0, 8, 0, 509, 0, 0, 2750, 0, 4059, 0, 1, 1466, 8, 61, 4, 92, 0, 0, 1, 0, 668, 2, 0, 0, 0, 854, 3426, 0, 0, 14, 1, 0, 37, 1848, 0, 0, 1, 9, 0, 1, 0, 4085, 0, 0, 197, 0, 418, 0, 0, 0, 0, 465, 0] payload_rows_per_dst=[8192, 8152, 8134, 2698, 6856, 3922, 8180, 1530] dup_rows_per_dst=[7616, 640, 8434, 478, 0, 64, 404, 236] combine_rows_per_src=[8192, 8152, 8134, 2698, 6856, 3922, 8180, 1530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54778 payload_rows_remote=46788 dup_rows=10758 dup_rows_remote=9978 dedup_ratio=0.164154 router_tokens_per_expert=[0, 0, 0, 4080, 2, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 117, 0, 0, 14, 0, 293, 4093, 0, 4067, 8, 0, 1, 0, 0, 145, 3985, 3992, 33, 1, 427, 0, 0, 0, 541, 0, 0, 0, 0, 4096, 0, 132, 0, 0, 0, 0, 108, 11, 223, 0, 15, 80, 0, 0, 2170] payload_rows_per_dst=[7990, 8160, 1162, 8138, 8190, 8186, 8126, 4826] dup_rows_per_dst=[780, 0, 74, 7112, 1232, 1082, 306, 172] combine_rows_per_src=[7990, 8160, 1162, 8138, 8190, 8186, 8126, 4826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53298 payload_rows_remote=47966 dup_rows=12238 dup_rows_remote=12102 dedup_ratio=0.186737 router_tokens_per_expert=[1, 0, 2, 3616, 0, 0, 4083, 2, 0, 47, 7, 0, 4, 0, 16, 0, 12, 388, 1, 3880, 0, 0, 0, 2, 595, 131, 41, 0, 4, 1922, 0, 38, 0, 0, 0, 0, 3586, 2860, 0, 0, 0, 34, 0, 0, 0, 0, 4, 2524, 4030, 0, 0, 502, 0, 0, 7, 2340, 2, 6, 0, 0, 0, 0, 0, 2081] payload_rows_per_dst=[8172, 7258, 7790, 4068, 7316, 5332, 5300, 8062] dup_rows_per_dst=[4, 122, 776, 1050, 5576, 136, 912, 3662] combine_rows_per_src=[8172, 7258, 7790, 4068, 7316, 5332, 5300, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52980 payload_rows_remote=48296 dup_rows=12556 dup_rows_remote=12532 dedup_ratio=0.191589 router_tokens_per_expert=[0, 0, 2798, 3859, 62, 0, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2322, 17, 0, 0, 0, 575, 1, 4094, 0, 0, 0, 0, 358, 0, 0, 1, 0, 3656, 0, 1, 1, 0, 0, 0, 0, 152, 0, 0, 0, 3293, 0, 4061, 0, 0, 0, 0, 67, 5, 59, 244, 257, 200, 0, 3, 3931] payload_rows_per_dst=[7892, 7718, 4684, 8178, 4, 8188, 8192, 8124] dup_rows_per_dst=[3300, 0, 24, 1002, 2, 304, 6650, 1274] combine_rows_per_src=[7892, 7718, 4684, 8178, 4, 8188, 8192, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=170123264 dispatch_meta_bytes=181360 combine_payload_bytes=170123264 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=170304624 ingress_bytes=245620736 link_bytes=245620736 token_bytes=340427888 weight_movement_bytes=75497472 total_nvlink_bytes=415925360 weight_to_token_ratio=0.221772 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=8192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.868890002e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=156409856 dispatch_meta_bytes=194768 combine_payload_bytes=156409856 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=156604624 ingress_bytes=181575680 link_bytes=181575680 token_bytes=313014480 weight_movement_bytes=25165824 total_nvlink_bytes=338180304 weight_to_token_ratio=0.080398 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.381581126e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=161841152 dispatch_meta_bytes=226976 combine_payload_bytes=161841152 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=162068128 ingress_bytes=212172800 link_bytes=212172800 token_bytes=323909280 weight_movement_bytes=50331648 total_nvlink_bytes=374240928 weight_to_token_ratio=0.155388 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.614389856e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=191643648 dispatch_meta_bytes=227064 combine_payload_bytes=191643648 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=191870712 ingress_bytes=267141120 link_bytes=267141120 token_bytes=383514360 weight_movement_bytes=75497472 total_nvlink_bytes=459011832 weight_to_token_ratio=0.196857 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.032635259e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=180969472 dispatch_meta_bytes=235768 combine_payload_bytes=180969472 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=181205240 ingress_bytes=281632768 link_bytes=281632768 token_bytes=362174712 weight_movement_bytes=100663296 total_nvlink_bytes=462838008 weight_to_token_ratio=0.277941 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.142899956e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=196468736 dispatch_meta_bytes=240272 combine_payload_bytes=196468736 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=196709008 ingress_bytes=246800384 link_bytes=246800384 token_bytes=393177744 weight_movement_bytes=50331648 total_nvlink_bytes=443509392 weight_to_token_ratio=0.128012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=13824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.877865760e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=197820416 dispatch_meta_bytes=243312 combine_payload_bytes=197820416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198063728 ingress_bytes=222986240 link_bytes=222986240 token_bytes=395884144 weight_movement_bytes=25165824 total_nvlink_bytes=421049968 weight_to_token_ratio=0.063569 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.696667640e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=42858 payload_rows_remote=34708 dup_rows=22678 dup_rows_remote=22672 dedup_ratio=0.346039 router_tokens_per_expert=[0, 0, 73, 3634, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 1, 535, 4066, 2556, 0, 1, 0, 0, 4066, 327, 0, 9, 1, 0, 33, 0, 0, 0, 0, 4063, 0, 1, 14, 0, 33, 0, 6, 3294, 0, 0, 0, 4054, 0, 2096, 22, 0, 0, 0, 3667, 3, 0, 4, 14, 63, 0, 0, 25] payload_rows_per_dst=[150, 7268, 8192, 3272, 8150, 7450, 8186, 190] dup_rows_per_dst=[0, 0, 6336, 42, 6, 4774, 11492, 28] combine_rows_per_src=[150, 7268, 8192, 3272, 8150, 7450, 8186, 190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=142163968 dispatch_meta_bytes=229520 combine_payload_bytes=142163968 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=142393488 ingress_bytes=242827264 link_bytes=242827264 token_bytes=284557456 weight_movement_bytes=100663296 total_nvlink_bytes=385220752 weight_to_token_ratio=0.353754 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.847634908e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61876 payload_rows_remote=54082 dup_rows=3660 dup_rows_remote=696 dedup_ratio=0.055847 router_tokens_per_expert=[0, 4020, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3873, 17, 0, 4076, 76, 0, 0, 0, 0, 0, 117, 0, 0, 135, 4045, 22, 33, 2, 1, 289, 0, 0, 0, 0, 4083, 0, 0, 1, 3751, 0, 1, 0, 3, 0, 11, 0, 0, 6] payload_rows_per_dst=[8040, 8162, 8178, 7794, 7506, 5858, 8166, 8172] dup_rows_per_dst=[60, 58, 0, 2964, 500, 54, 2, 22] combine_rows_per_src=[8040, 8162, 8178, 7794, 7506, 5858, 8166, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47412 payload_rows_remote=39220 dup_rows=18124 dup_rows_remote=9832 dedup_ratio=0.276550 router_tokens_per_expert=[0, 69, 0, 0, 0, 0, 0, 266, 0, 315, 0, 1, 1, 4028, 2, 838, 0, 5, 3, 0, 72, 0, 95, 0, 0, 53, 3970, 26, 0, 3885, 0, 0, 295, 1, 0, 0, 4006, 121, 0, 0, 2936, 0, 3895, 0, 3004, 0, 0, 578, 0, 0, 0, 2, 27, 6, 4091, 0, 0, 0, 164, 0, 12, 1, 0, 0] payload_rows_per_dst=[6482, 8192, 1956, 7964, 8182, 6766, 68, 7802] dup_rows_per_dst=[60, 8292, 70, 134, 8846, 398, 2, 322] combine_rows_per_src=[6482, 8192, 1956, 7964, 8182, 6766, 68, 7802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57874 payload_rows_remote=49682 dup_rows=7662 dup_rows_remote=7600 dedup_ratio=0.116913 router_tokens_per_expert=[0, 3664, 0, 0, 0, 363, 0, 0, 0, 40, 0, 0, 7, 3759, 1, 4078, 0, 0, 117, 14, 11, 0, 47, 0, 105, 0, 542, 0, 0, 4096, 419, 0, 0, 267, 2, 0, 4081, 0, 0, 56, 6, 33, 3948, 2, 0, 7, 0, 12, 0, 0, 3960, 0, 0, 206, 2894, 0, 18, 10, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[7654, 7606, 8190, 1950, 8176, 7990, 8116, 8192] dup_rows_per_dst=[400, 8, 344, 182, 636, 26, 6004, 62] combine_rows_per_src=[7654, 7606, 8190, 1950, 8176, 7990, 8116, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49802 payload_rows_remote=41652 dup_rows=15734 dup_rows_remote=10618 dedup_ratio=0.240082 router_tokens_per_expert=[0, 2667, 0, 0, 0, 73, 0, 0, 0, 0, 1, 0, 5, 0, 4, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 3907, 1, 0, 4079, 30, 0, 17, 626, 0, 51, 114, 0, 4, 885, 3893, 6, 24, 20, 109, 4034, 0, 0, 0, 0, 4088, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8150, 1944, 6250, 8170, 8192, 8174, 6746, 2176] dup_rows_per_dst=[5116, 68, 0, 5696, 3260, 1594, 0, 0] combine_rows_per_src=[8150, 1944, 6250, 8170, 8192, 8174, 6746, 2176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57488 payload_rows_remote=49324 dup_rows=8048 dup_rows_remote=3090 dedup_ratio=0.122803 router_tokens_per_expert=[10, 1329, 0, 0, 0, 0, 0, 0, 230, 8, 52, 0, 0, 4032, 30, 4060, 14, 0, 1, 0, 0, 9, 0, 0, 0, 1, 2710, 16, 0, 4087, 770, 0, 2, 0, 0, 0, 3531, 18, 0, 155, 1848, 0, 2531, 6, 0, 2044, 132, 0, 9, 1, 0, 0, 19, 7, 4037, 0, 0, 0, 122, 0, 0, 947, 0, 0] payload_rows_per_dst=[2678, 8192, 8162, 6770, 7202, 8164, 8130, 8190] dup_rows_per_dst=[0, 512, 6, 224, 210, 4958, 16, 2122] combine_rows_per_src=[2678, 8192, 8162, 6770, 7202, 8164, 8130, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54230 payload_rows_remote=46038 dup_rows=11306 dup_rows_remote=11124 dedup_ratio=0.172516 router_tokens_per_expert=[0, 3339, 0, 0, 0, 8, 0, 0, 0, 258, 5, 1, 49, 3970, 22, 4096, 0, 0, 0, 0, 0, 0, 91, 0, 0, 2, 110, 226, 0, 4073, 26, 0, 0, 0, 0, 0, 213, 0, 1, 0, 4058, 0, 2, 0, 7, 1288, 0, 0, 0, 0, 3938, 0, 0, 894, 2878, 0, 0, 0, 0, 0, 0, 3213, 0, 0] payload_rows_per_dst=[8070, 8120, 8192, 5104, 6114, 2586, 7876, 8168] dup_rows_per_dst=[2248, 494, 182, 116, 70, 4, 1788, 6404] combine_rows_per_src=[8070, 8120, 8192, 5104, 6114, 2586, 7876, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55282 payload_rows_remote=47090 dup_rows=10254 dup_rows_remote=9668 dedup_ratio=0.156464 router_tokens_per_expert=[0, 0, 682, 0, 3005, 3904, 0, 2668, 0, 3296, 0, 0, 2, 0, 11, 4080, 23, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 14, 1281, 4096, 811, 2170, 0, 0, 0, 65, 0, 0, 0, 2, 0, 0, 1328, 213, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 0] payload_rows_per_dst=[6512, 8192, 5388, 8184, 4104, 6658, 8192, 8052] dup_rows_per_dst=[862, 6586, 6, 30, 210, 768, 586, 1206] combine_rows_per_src=[6512, 8192, 5388, 8184, 4104, 6658, 8192, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=202031104 dispatch_meta_bytes=209656 combine_payload_bytes=202031104 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=202240760 ingress_bytes=302694400 link_bytes=302694400 token_bytes=404271864 weight_movement_bytes=100663296 total_nvlink_bytes=504935160 weight_to_token_ratio=0.248999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.303154640e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=203497472 dispatch_meta_bytes=229128 combine_payload_bytes=203497472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203726600 ingress_bytes=228663296 link_bytes=228663296 token_bytes=407224072 weight_movement_bytes=25165824 total_nvlink_bytes=432389896 weight_to_token_ratio=0.061798 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.739863477e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=221519872 dispatch_meta_bytes=219112 combine_payload_bytes=221519872 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=221738984 ingress_bytes=271851520 link_bytes=271851520 token_bytes=443258856 weight_movement_bytes=50331648 total_nvlink_bytes=493590504 weight_to_token_ratio=0.113549 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=23296 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.068475960e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160645120 dispatch_meta_bytes=196208 combine_payload_bytes=160645120 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160841328 ingress_bytes=210976768 link_bytes=210976768 token_bytes=321486448 weight_movement_bytes=50331648 total_nvlink_bytes=371818096 weight_to_token_ratio=0.156559 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.605289434e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=170606592 dispatch_meta_bytes=209080 combine_payload_bytes=170606592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=170815672 ingress_bytes=195772416 link_bytes=195772416 token_bytes=341422264 weight_movement_bytes=25165824 total_nvlink_bytes=366588088 weight_to_token_ratio=0.073709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=23040 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.489601883e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=188571648 dispatch_meta_bytes=228648 combine_payload_bytes=188571648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188800296 ingress_bytes=213737472 link_bytes=213737472 token_bytes=377371944 weight_movement_bytes=25165824 total_nvlink_bytes=402537768 weight_to_token_ratio=0.066687 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.626295202e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60240 payload_rows_remote=54280 dup_rows=5296 dup_rows_remote=4614 dedup_ratio=0.080811 router_tokens_per_expert=[128, 4092, 0, 0, 0, 12, 0, 0, 0, 0, 2, 1, 24, 3587, 10, 4091, 226, 0, 0, 0, 0, 197, 0, 0, 0, 7, 4055, 127, 0, 3973, 0, 0, 13, 0, 0, 0, 44, 0, 0, 2676, 269, 0, 3583, 0, 9, 870, 0, 0, 0, 0, 166, 12, 0, 3326, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7246, 8186, 8192, 5960, 7186, 7334, 7946] dup_rows_per_dst=[274, 2, 842, 186, 682, 2276, 1034, 0] combine_rows_per_src=[8190, 7246, 8186, 8192, 5960, 7186, 7334, 7946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=192880640 dispatch_meta_bytes=227032 combine_payload_bytes=192880640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193107672 ingress_bytes=218046464 link_bytes=218046464 token_bytes=385988312 weight_movement_bytes=25165824 total_nvlink_bytes=411154136 weight_to_token_ratio=0.065198 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.659081652e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=222330880 dispatch_meta_bytes=235576 combine_payload_bytes=222330880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=222566456 ingress_bytes=247496704 link_bytes=247496704 token_bytes=444897336 weight_movement_bytes=25165824 total_nvlink_bytes=470063160 weight_to_token_ratio=0.056565 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.883163951e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52600 payload_rows_remote=44898 dup_rows=12936 dup_rows_remote=12702 dedup_ratio=0.197388 router_tokens_per_expert=[0, 50, 311, 8, 0, 429, 3175, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 49, 332, 0, 1, 0, 3824, 1465, 0, 0, 0, 0, 465, 3977, 1, 0, 0, 4008, 0, 0, 0, 0, 11, 272, 0, 0, 117, 0, 0, 0, 27, 0, 0, 0, 3905, 1, 0, 0, 0, 1394, 3134, 0, 0, 0, 3964, 1566, 0, 7] payload_rows_per_dst=[1390, 8188, 3416, 7984, 8192, 7702, 7920, 7808] dup_rows_per_dst=[206, 290, 278, 902, 390, 234, 2680, 7956] combine_rows_per_src=[1390, 8188, 3416, 7984, 8192, 7702, 7920, 7808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52952 payload_rows_remote=51922 dup_rows=12584 dup_rows_remote=12584 dedup_ratio=0.192017 router_tokens_per_expert=[0, 0, 0, 0, 4095, 144, 20, 0, 0, 0, 50, 4, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 2506, 4095, 0, 0, 0, 441, 0, 0, 74, 0, 0, 3978, 90, 2, 0, 0, 0, 1086, 0, 0, 205, 2, 0, 0, 0, 0, 7, 1800, 985, 0, 0, 0, 0, 1650, 2, 0, 2, 34, 3972, 4082, 3438, 0] payload_rows_per_dst=[8190, 7954, 8190, 1030, 8178, 5036, 6186, 8188] dup_rows_per_dst=[328, 100, 1976, 0, 2134, 390, 728, 6928] combine_rows_per_src=[8190, 7954, 8190, 1030, 8178, 5036, 6186, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49680 payload_rows_remote=41488 dup_rows=15856 dup_rows_remote=14974 dedup_ratio=0.241943 router_tokens_per_expert=[0, 0, 118, 105, 0, 0, 4015, 1, 1, 244, 2, 0, 198, 0, 0, 1, 0, 3771, 0, 0, 0, 0, 3994, 86, 3805, 0, 336, 0, 8, 5, 935, 0, 3263, 583, 7, 0, 0, 0, 2, 56, 0, 0, 3690, 0, 0, 3, 0, 0, 0, 0, 2141, 0, 66, 0, 4, 109, 15, 0, 1110, 0, 4091, 0, 3, 0] payload_rows_per_dst=[8030, 8192, 7714, 7960, 7422, 8000, 138, 2224] dup_rows_per_dst=[448, 882, 4282, 2436, 400, 7374, 2, 32] combine_rows_per_src=[8030, 8192, 7714, 7960, 7422, 8000, 138, 2224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47958 payload_rows_remote=39768 dup_rows=17578 dup_rows_remote=9606 dedup_ratio=0.268219 router_tokens_per_expert=[0, 0, 14, 3640, 3353, 0, 725, 349, 0, 0, 0, 0, 0, 0, 122, 13, 0, 2, 0, 0, 0, 0, 436, 3977, 7, 0, 0, 0, 3680, 0, 224, 0, 0, 3755, 2, 0, 0, 0, 566, 0, 0, 0, 2964, 0, 279, 0, 0, 0, 479, 0, 443, 0, 48, 0, 0, 3589, 0, 0, 0, 0, 4090, 11, 0, 0] payload_rows_per_dst=[8190, 8190, 7960, 8182, 7792, 6568, 1054, 22] dup_rows_per_dst=[7972, 260, 884, 6818, 854, 790, 0, 0] combine_rows_per_src=[8190, 8190, 7960, 8182, 7792, 6568, 1054, 22] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=40644 payload_rows_remote=32790 dup_rows=24892 dup_rows_remote=19390 dedup_ratio=0.379822 router_tokens_per_expert=[2, 0, 0, 2, 3583, 719, 1632, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 39, 0, 1, 4064, 1368, 0, 0, 0, 62, 2, 1, 1240, 0, 47, 3769, 0, 16, 4, 0, 166, 0, 0, 0, 0, 0, 238, 4, 0, 0, 0, 0, 4089, 0, 0, 0, 9, 4090, 0, 0, 0, 0, 3672, 3919, 0, 3] payload_rows_per_dst=[4510, 5096, 78, 2518, 4260, 8138, 8190, 7854] dup_rows_per_dst=[1370, 0, 4, 92, 156, 3210, 14558, 5502] combine_rows_per_src=[4510, 5096, 78, 2518, 4260, 8138, 8190, 7854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51378 payload_rows_remote=43190 dup_rows=14158 dup_rows_remote=8868 dedup_ratio=0.216034 router_tokens_per_expert=[0, 1, 236, 1893, 6, 0, 4042, 214, 0, 0, 0, 0, 0, 0, 932, 0, 0, 1708, 181, 252, 2, 0, 2392, 665, 44, 0, 0, 83, 0, 2, 94, 0, 0, 3134, 2, 0, 1159, 0, 5, 0, 0, 0, 332, 0, 21, 0, 0, 6, 0, 2, 3931, 0, 3, 0, 0, 3966, 0, 0, 0, 25, 4016, 1955, 1464, 0] payload_rows_per_dst=[8192, 8032, 8188, 3458, 7336, 4838, 4488, 6846] dup_rows_per_dst=[4592, 1864, 5290, 442, 1264, 664, 0, 42] combine_rows_per_src=[8192, 8032, 8188, 3458, 7336, 4838, 4488, 6846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=183902208 dispatch_meta_bytes=230400 combine_payload_bytes=183902208 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=184132608 ingress_bytes=234233856 link_bytes=234233856 token_bytes=368034816 weight_movement_bytes=50331648 total_nvlink_bytes=418366464 weight_to_token_ratio=0.136758 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782249002e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=212672512 dispatch_meta_bytes=258024 combine_payload_bytes=212672512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=212930536 ingress_bytes=237838336 link_bytes=237838336 token_bytes=425603048 weight_movement_bytes=25165824 total_nvlink_bytes=450768872 weight_to_token_ratio=0.059130 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.809674930e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=169934848 dispatch_meta_bytes=225848 combine_payload_bytes=169934848 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=170160696 ingress_bytes=195100672 link_bytes=195100672 token_bytes=340095544 weight_movement_bytes=25165824 total_nvlink_bytes=365261368 weight_to_token_ratio=0.073996 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.484490688e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56340 payload_rows_remote=48194 dup_rows=9196 dup_rows_remote=9056 dedup_ratio=0.140320 router_tokens_per_expert=[0, 0, 4, 77, 2, 0, 4095, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4092, 3585, 2, 0, 0, 6, 0, 897, 3655, 0, 0, 3966, 92, 0, 0, 0, 85, 0, 0, 0, 257, 0, 1195, 0, 0, 0, 0, 3889, 2638, 0, 0, 0, 0, 5, 0, 0, 0, 15, 4004, 41, 0, 0] payload_rows_per_dst=[3400, 8008, 7692, 8170, 8146, 8190, 7830, 4904] dup_rows_per_dst=[494, 0, 1188, 950, 140, 2898, 3526, 0] combine_rows_per_src=[3400, 8008, 7692, 8170, 8146, 8190, 7830, 4904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=162889728 dispatch_meta_bytes=197496 combine_payload_bytes=162889728 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=163087224 ingress_bytes=213221376 link_bytes=213221376 token_bytes=325976952 weight_movement_bytes=50331648 total_nvlink_bytes=376308600 weight_to_token_ratio=0.154402 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.622368308e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=134307840 dispatch_meta_bytes=208720 combine_payload_bytes=134307840 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=134516560 ingress_bytes=184639488 link_bytes=184639488 token_bytes=268824400 weight_movement_bytes=50331648 total_nvlink_bytes=319156048 weight_to_token_ratio=0.187229 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=14848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404893165e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=176906240 dispatch_meta_bytes=208232 combine_payload_bytes=176906240 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=177114472 ingress_bytes=252403712 link_bytes=252403712 token_bytes=354020712 weight_movement_bytes=75497472 total_nvlink_bytes=429518184 weight_to_token_ratio=0.213257 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17408 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.920500612e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50056 payload_rows_remote=41864 dup_rows=15480 dup_rows_remote=15072 dedup_ratio=0.236206 router_tokens_per_expert=[31, 0, 4, 1, 2, 0, 0, 0, 0, 0, 58, 14, 0, 0, 0, 4014, 0, 246, 4025, 4095, 0, 204, 4096, 0, 0, 0, 662, 3828, 0, 0, 0, 1208, 0, 0, 0, 0, 0, 0, 0, 1, 3071, 2, 38, 1, 0, 0, 125, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 13, 463, 0] payload_rows_per_dst=[76, 8148, 3302, 8122, 8190, 8178, 8192, 5848] dup_rows_per_dst=[0, 24, 360, 8154, 2, 6466, 408, 66] combine_rows_per_src=[76, 8148, 3302, 8122, 8190, 8178, 8192, 5848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=197402624 dispatch_meta_bytes=229000 combine_payload_bytes=197402624 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=197631624 ingress_bytes=247734272 link_bytes=247734272 token_bytes=395034248 weight_movement_bytes=50331648 total_nvlink_bytes=445365896 weight_to_token_ratio=0.127411 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=22656 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.884971569e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=171474944 dispatch_meta_bytes=227744 combine_payload_bytes=171474944 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=171702688 ingress_bytes=221806592 link_bytes=221806592 token_bytes=343177632 weight_movement_bytes=50331648 total_nvlink_bytes=393509280 weight_to_token_ratio=0.146664 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=22528 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.687691882e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47084 payload_rows_remote=39174 dup_rows=18452 dup_rows_remote=16486 dedup_ratio=0.281555 router_tokens_per_expert=[4, 0, 0, 0, 104, 0, 90, 0, 38, 670, 0, 0, 0, 4091, 0, 495, 20, 3319, 0, 3821, 7, 0, 3988, 0, 1, 317, 0, 0, 0, 0, 0, 0, 3630, 1864, 0, 0, 0, 0, 0, 174, 1160, 0, 24, 74, 43, 4, 0, 3, 0, 124, 4040, 551, 0, 1, 0, 6, 0, 49, 0, 0, 4045, 7, 4, 0] payload_rows_per_dst=[188, 2416, 8036, 8186, 4076, 7910, 8100, 8172] dup_rows_per_dst=[0, 198, 13070, 632, 0, 1966, 2548, 38] combine_rows_per_src=[188, 2416, 8036, 8186, 4076, 7910, 8100, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57714 payload_rows_remote=49522 dup_rows=7822 dup_rows_remote=7820 dedup_ratio=0.119354 router_tokens_per_expert=[0, 0, 0, 17, 2934, 130, 0, 0, 0, 4095, 259, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 75, 0, 1, 0, 0, 0, 4041, 0, 0, 2630, 11, 0, 0, 0, 0, 0, 1251, 5, 0, 4, 0, 0, 13, 35, 0, 0, 0, 4096, 0, 0, 384] payload_rows_per_dst=[6094, 8190, 8152, 8192, 8186, 8188, 2520, 8192] dup_rows_per_dst=[68, 518, 1040, 2, 48, 5282, 26, 838] combine_rows_per_src=[6094, 8190, 8152, 8192, 8186, 8188, 2520, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47060 payload_rows_remote=39050 dup_rows=18476 dup_rows_remote=18018 dedup_ratio=0.281921 router_tokens_per_expert=[40, 0, 0, 0, 1556, 0, 0, 1, 0, 3587, 8, 84, 380, 4065, 0, 175, 1318, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 1, 0, 0, 0, 0, 4095, 24, 4023, 0, 3838, 0, 0, 4078, 1554, 0, 128, 0, 0, 0, 0, 0, 1, 0, 21, 2502, 0, 0, 0, 807, 0, 0, 0, 0, 467, 0, 0, 0] payload_rows_per_dst=[4398, 8010, 2786, 8154, 8190, 8046, 6542, 934] dup_rows_per_dst=[88, 458, 0, 6, 17262, 542, 120, 0] combine_rows_per_src=[4398, 8010, 2786, 8154, 8190, 8046, 6542, 934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50142 payload_rows_remote=44510 dup_rows=15394 dup_rows_remote=15394 dedup_ratio=0.234894 router_tokens_per_expert=[0, 0, 2810, 1, 0, 0, 0, 0, 0, 1442, 0, 0, 4, 4096, 0, 0, 1007, 197, 0, 35, 3944, 1975, 6, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 5, 345, 0, 0, 0, 0, 0, 0, 31, 289, 0, 4079, 24, 0, 0, 0, 147, 0, 3547, 0, 718, 3938, 0, 29, 0] payload_rows_per_dst=[5632, 3886, 8192, 7198, 8186, 752, 8192, 8104] dup_rows_per_dst=[0, 12, 6136, 0, 0, 0, 886, 8360] combine_rows_per_src=[5632, 3886, 8192, 7198, 8186, 752, 8192, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50104 payload_rows_remote=41996 dup_rows=15432 dup_rows_remote=8032 dedup_ratio=0.235474 router_tokens_per_expert=[0, 0, 3899, 3304, 4007, 61, 1, 0, 18, 3430, 0, 0, 78, 3984, 24, 0, 664, 1, 0, 253, 531, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 98, 3938, 0, 0, 0, 90, 0, 0, 1629, 0, 0, 0, 146, 0, 0, 0, 23, 3, 0, 0, 0, 0, 0, 0, 2838, 0, 0, 0, 0, 3747, 0, 0, 0] payload_rows_per_dst=[8144, 7056, 1418, 8160, 3422, 8112, 5684, 8108] dup_rows_per_dst=[6386, 44, 1478, 6, 16, 102, 0, 7400] combine_rows_per_src=[8144, 7056, 1418, 8160, 3422, 8112, 5684, 8108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56170 payload_rows_remote=48484 dup_rows=9366 dup_rows_remote=9356 dedup_ratio=0.142914 router_tokens_per_expert=[0, 0, 1606, 1554, 3768, 292, 0, 0, 0, 3439, 0, 0, 321, 4096, 0, 0, 10, 7, 0, 9, 3820, 0, 2, 0, 0, 0, 0, 0, 0, 211, 0, 0, 4080, 0, 0, 0, 247, 0, 0, 28, 0, 0, 0, 0, 5, 0, 0, 27, 24, 0, 3823, 6, 0, 0, 0, 661, 0, 2147, 0, 0, 2571, 0, 0, 14] payload_rows_per_dst=[8062, 7518, 7686, 8192, 494, 8178, 8188, 7852] dup_rows_per_dst=[6378, 2, 10, 422, 56, 46, 840, 1612] combine_rows_per_src=[8062, 7518, 7686, 8192, 494, 8178, 8188, 7852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160456704 dispatch_meta_bytes=222640 combine_payload_bytes=160456704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160679344 ingress_bytes=185622528 link_bytes=185622528 token_bytes=321136048 weight_movement_bytes=25165824 total_nvlink_bytes=346301872 weight_to_token_ratio=0.078365 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412372963e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=202842112 dispatch_meta_bytes=229368 combine_payload_bytes=202842112 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203071480 ingress_bytes=228007936 link_bytes=228007936 token_bytes=405913592 weight_movement_bytes=25165824 total_nvlink_bytes=431079416 weight_to_token_ratio=0.061998 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.734876944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48726 payload_rows_remote=41500 dup_rows=16810 dup_rows_remote=16808 dedup_ratio=0.256500 router_tokens_per_expert=[0, 0, 4078, 0, 4035, 0, 3, 0, 0, 3963, 0, 0, 1281, 4096, 3, 0, 0, 115, 0, 50, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4063, 0, 1, 10, 0, 0, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 461, 98, 2773, 11, 3, 0, 0, 0, 3921, 0, 13, 0, 0, 155, 0, 30, 0] payload_rows_per_dst=[8156, 8100, 296, 8192, 7226, 8190, 8180, 386] dup_rows_per_dst=[5146, 5324, 36, 2, 2, 858, 5432, 10] combine_rows_per_src=[8156, 8100, 296, 8192, 7226, 8190, 8180, 386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159948800 dispatch_meta_bytes=228272 combine_payload_bytes=159948800 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160177072 ingress_bytes=210280448 link_bytes=210280448 token_bytes=320125872 weight_movement_bytes=50331648 total_nvlink_bytes=370457520 weight_to_token_ratio=0.157225 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599991244e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=182312960 dispatch_meta_bytes=239616 combine_payload_bytes=182312960 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=182552576 ingress_bytes=257810432 link_bytes=257810432 token_bytes=364865536 weight_movement_bytes=75497472 total_nvlink_bytes=440363008 weight_to_token_ratio=0.206919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.961639504e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=172015616 dispatch_meta_bytes=200112 combine_payload_bytes=172015616 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=172215728 ingress_bytes=197181440 link_bytes=197181440 token_bytes=344231344 weight_movement_bytes=25165824 total_nvlink_bytes=369397168 weight_to_token_ratio=0.073107 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.500322928e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50572 payload_rows_remote=50476 dup_rows=14964 dup_rows_remote=14920 dedup_ratio=0.228333 router_tokens_per_expert=[0, 7, 795, 3949, 0, 0, 4010, 0, 21, 0, 0, 0, 0, 0, 0, 202, 0, 3, 197, 4055, 195, 0, 0, 0, 0, 1, 0, 0, 13, 642, 0, 2455, 0, 0, 0, 1478, 4096, 0, 0, 0, 0, 20, 158, 0, 678, 364, 0, 1386, 4008, 39, 0, 17, 0, 0, 14, 0, 0, 7, 0, 1, 0, 0, 0, 3957] payload_rows_per_dst=[8032, 8098, 8154, 5100, 8192, 4854, 96, 8046] dup_rows_per_dst=[1592, 246, 746, 1120, 2956, 360, 44, 7900] combine_rows_per_src=[8032, 8098, 8154, 5100, 8192, 4854, 96, 8046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=198590464 dispatch_meta_bytes=231360 combine_payload_bytes=198590464 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=198821824 ingress_bytes=248922112 link_bytes=248922112 token_bytes=397412288 weight_movement_bytes=50331648 total_nvlink_bytes=447743936 weight_to_token_ratio=0.126648 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=9088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.894009659e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=169984000 dispatch_meta_bytes=233232 combine_payload_bytes=169984000 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=170217232 ingress_bytes=245481472 link_bytes=245481472 token_bytes=340201232 weight_movement_bytes=75497472 total_nvlink_bytes=415698704 weight_to_token_ratio=0.221920 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.867830364e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=206749696 dispatch_meta_bytes=261584 combine_payload_bytes=206749696 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=207011280 ingress_bytes=257081344 link_bytes=257081344 token_bytes=413760976 weight_movement_bytes=50331648 total_nvlink_bytes=464092624 weight_to_token_ratio=0.121644 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.956091987e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53730 payload_rows_remote=46858 dup_rows=11806 dup_rows_remote=10564 dedup_ratio=0.180145 router_tokens_per_expert=[0, 124, 6, 53, 4096, 0, 0, 0, 0, 59, 0, 141, 2158, 3890, 44, 3, 0, 0, 0, 0, 5, 3662, 0, 9, 2468, 13, 0, 276, 41, 4077, 238, 0, 323, 2, 0, 3375, 0, 1759, 0, 20, 0, 1, 1, 0, 3185, 0, 870, 0, 1585, 1, 0, 18, 0, 0, 150, 112, 0, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4604, 7346, 8192, 7182, 6872, 3556, 7786] dup_rows_per_dst=[366, 82, 6, 6034, 3894, 1242, 182, 0] combine_rows_per_src=[8192, 4604, 7346, 8192, 7182, 6872, 3556, 7786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57498 payload_rows_remote=50198 dup_rows=8038 dup_rows_remote=8030 dedup_ratio=0.122650 router_tokens_per_expert=[0, 3, 341, 41, 3940, 0, 0, 0, 4, 2247, 72, 0, 2710, 4059, 0, 4096, 0, 0, 0, 0, 0, 3857, 0, 0, 2840, 0, 2, 812, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 317, 3, 0, 0, 0, 0, 56, 31, 0, 2627, 0, 0, 0, 628, 23, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5572, 7714, 7300, 8106, 8022, 4474, 8118] dup_rows_per_dst=[458, 152, 0, 8, 5120, 2292, 8, 0] combine_rows_per_src=[8192, 5572, 7714, 7300, 8106, 8022, 4474, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44768 payload_rows_remote=36576 dup_rows=20768 dup_rows_remote=18796 dedup_ratio=0.316895 router_tokens_per_expert=[12, 0, 76, 0, 4096, 0, 127, 771, 0, 4093, 3541, 2, 0, 4093, 0, 3925, 0, 0, 0, 0, 0, 4096, 0, 0, 97, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1714, 0, 1, 0, 0, 0, 0, 0, 241, 14, 0, 0, 1, 67, 1847, 558, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 194, 3430, 500, 7888, 8186] dup_rows_per_dst=[1972, 14936, 0, 4, 0, 12, 3844, 0] combine_rows_per_src=[8192, 8186, 8192, 194, 3430, 500, 7888, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47372 payload_rows_remote=45304 dup_rows=18164 dup_rows_remote=18092 dedup_ratio=0.277161 router_tokens_per_expert=[0, 0, 8, 3387, 2328, 0, 0, 0, 18, 2943, 17, 0, 20, 0, 1, 285, 0, 0, 0, 0, 66, 4087, 0, 0, 3914, 85, 0, 0, 0, 3957, 80, 0, 0, 3702, 0, 44, 0, 244, 0, 1, 0, 4, 0, 3569, 0, 0, 0, 0, 6, 3126, 153, 17, 2, 0, 0, 563, 0, 0, 0, 141, 0, 0, 0, 0] payload_rows_per_dst=[7132, 2068, 8184, 8124, 7864, 7146, 6572, 282] dup_rows_per_dst=[4354, 72, 122, 7948, 4506, 0, 1162, 0] combine_rows_per_src=[7132, 2068, 8184, 8124, 7864, 7146, 6572, 282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47122 payload_rows_remote=38930 dup_rows=18414 dup_rows_remote=11858 dedup_ratio=0.280975 router_tokens_per_expert=[0, 27, 1, 366, 4013, 0, 0, 0, 0, 34, 4076, 0, 447, 3876, 25, 3942, 4, 0, 0, 0, 1522, 3936, 0, 0, 0, 133, 0, 0, 2, 0, 10, 0, 0, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 2082, 4068, 127, 799, 0, 0, 6, 0, 206, 0, 0, 0, 0, 2919, 0, 0, 129, 0, 0, 0, 3] payload_rows_per_dst=[732, 5668, 7880, 8190, 98, 8192, 8170, 8192] dup_rows_per_dst=[56, 12, 5566, 126, 0, 5960, 138, 6556] combine_rows_per_src=[732, 5668, 7880, 8190, 98, 8192, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55784 payload_rows_remote=47596 dup_rows=9752 dup_rows_remote=7428 dedup_ratio=0.148804 router_tokens_per_expert=[1530, 0, 0, 4, 4095, 0, 0, 45, 0, 2485, 3, 48, 278, 3107, 0, 3865, 251, 0, 0, 0, 924, 4081, 0, 0, 2241, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2264, 6, 0, 0, 0, 0, 227, 4096, 0, 3, 0, 17, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7826, 8188, 4488, 6276, 8192, 6406, 6216] dup_rows_per_dst=[3440, 278, 2324, 16, 3234, 460, 0, 0] combine_rows_per_src=[8192, 7826, 8188, 4488, 6276, 8192, 6406, 6216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=191930368 dispatch_meta_bytes=229688 combine_payload_bytes=191930368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=192160056 ingress_bytes=217096192 link_bytes=217096192 token_bytes=384090424 weight_movement_bytes=25165824 total_nvlink_bytes=409256248 weight_to_token_ratio=0.065521 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.651851180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=205611008 dispatch_meta_bytes=232912 combine_payload_bytes=205611008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=205843920 ingress_bytes=230776832 link_bytes=230776832 token_bytes=411454928 weight_movement_bytes=25165824 total_nvlink_bytes=436620752 weight_to_token_ratio=0.061163 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.755945044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=149815296 dispatch_meta_bytes=221488 combine_payload_bytes=149815296 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=150036784 ingress_bytes=200146944 link_bytes=200146944 token_bytes=299852080 weight_movement_bytes=50331648 total_nvlink_bytes=350183728 weight_to_token_ratio=0.167855 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.522886987e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59904 payload_rows_remote=51750 dup_rows=5632 dup_rows_remote=5610 dedup_ratio=0.085938 router_tokens_per_expert=[12, 0, 370, 3, 4095, 0, 0, 0, 0, 1, 14, 19, 1668, 3853, 0, 3772, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 0, 15, 0, 0, 721, 0, 0, 4073, 3, 0, 10, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1674, 0, 29, 3, 149, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3402, 8192, 8188, 8154, 8192, 7876, 7708] dup_rows_per_dst=[768, 0, 0, 1416, 22, 3348, 30, 48] combine_rows_per_src=[8192, 3402, 8192, 8188, 8154, 8192, 7876, 7708] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=185565184 dispatch_meta_bytes=253584 combine_payload_bytes=185565184 prefetch_bytes=0 reduce_bytes=176160768 egress_bytes=185818768 ingress_bytes=361725952 link_bytes=361725952 token_bytes=371383952 weight_movement_bytes=176160768 total_nvlink_bytes=547544720 weight_to_token_ratio=0.474336 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=8576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.752316543e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159457280 dispatch_meta_bytes=203152 combine_payload_bytes=159457280 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=159660432 ingress_bytes=209788928 link_bytes=209788928 token_bytes=319117712 weight_movement_bytes=50331648 total_nvlink_bytes=369449360 weight_to_token_ratio=0.157721 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596251344e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50788 payload_rows_remote=44344 dup_rows=14748 dup_rows_remote=10938 dedup_ratio=0.225037 router_tokens_per_expert=[0, 3764, 265, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 1986, 1334, 4094, 0, 0, 0, 0, 0, 3395, 0, 0, 0, 0, 4093, 0, 0, 2553, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 562, 199, 2, 1, 3, 0, 0, 0, 0, 2649, 0, 0, 2301, 177, 0, 0, 7, 415, 0, 0, 0, 0, 4] payload_rows_per_dst=[7732, 6216, 8188, 8186, 0, 8174, 6444, 5848] dup_rows_per_dst=[382, 424, 6790, 1904, 0, 1328, 3810, 110] combine_rows_per_src=[7732, 6216, 8188, 8186, 0, 8174, 6444, 5848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=194953216 dispatch_meta_bytes=220096 combine_payload_bytes=194953216 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=195173312 ingress_bytes=220119040 link_bytes=220119040 token_bytes=390126528 weight_movement_bytes=25165824 total_nvlink_bytes=415292352 weight_to_token_ratio=0.064507 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.674851561e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=211968000 dispatch_meta_bytes=229440 combine_payload_bytes=211968000 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=212197440 ingress_bytes=237133824 link_bytes=237133824 token_bytes=424165440 weight_movement_bytes=25165824 total_nvlink_bytes=449331264 weight_to_token_ratio=0.059330 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=10880 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.804314408e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52696 payload_rows_remote=44842 dup_rows=12840 dup_rows_remote=12672 dedup_ratio=0.195923 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 3321, 0, 0, 210, 1110, 131, 0, 0, 0, 0, 2, 0, 0, 803, 71, 0, 22, 750, 3639, 0, 0, 1215, 3645, 0, 6, 9, 0, 0, 27, 315, 0, 0, 0, 0, 0, 1, 0, 87, 0, 0, 0, 278, 1, 0, 3968, 3888, 0, 0, 0, 0, 1, 5, 1193, 0, 3986, 0, 4083, 0] payload_rows_per_dst=[6644, 2800, 3032, 8110, 8192, 7854, 8080, 7984] dup_rows_per_dst=[0, 102, 264, 1628, 658, 168, 7634, 2386] combine_rows_per_src=[6644, 2800, 3032, 8110, 8192, 7854, 8080, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=181633024 dispatch_meta_bytes=221128 combine_payload_bytes=181633024 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=181854152 ingress_bytes=231964672 link_bytes=231964672 token_bytes=363487176 weight_movement_bytes=50331648 total_nvlink_bytes=413818824 weight_to_token_ratio=0.138469 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18176 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.764983134e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48098 payload_rows_remote=47992 dup_rows=17438 dup_rows_remote=17438 dedup_ratio=0.266083 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3241, 0, 1936, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 3538, 1227, 0, 0, 0, 3, 0, 0, 0, 4088, 0, 0, 50, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 1, 2067, 0, 574, 0, 0, 24, 0, 0, 0, 2415, 1224, 3, 4096, 0, 3958, 0] payload_rows_per_dst=[6482, 3872, 7998, 106, 8044, 8178, 5242, 8176] dup_rows_per_dst=[0, 336, 1538, 0, 116, 8066, 692, 6690] combine_rows_per_src=[6482, 3872, 7998, 106, 8044, 8178, 5242, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48698 payload_rows_remote=40506 dup_rows=16838 dup_rows_remote=9260 dedup_ratio=0.256927 router_tokens_per_expert=[0, 0, 2, 0, 0, 17, 3770, 0, 3998, 0, 0, 0, 0, 0, 118, 0, 0, 23, 0, 126, 4048, 0, 1, 0, 1, 107, 0, 0, 4086, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 248, 0, 0, 0, 4, 18, 0, 0, 184, 2604, 0, 15, 4069, 0, 0, 0, 0, 0, 2, 935, 0, 4096, 0, 4077, 201] payload_rows_per_dst=[8192, 8138, 8102, 8182, 8188, 412, 5208, 2276] dup_rows_per_dst=[7578, 1402, 294, 7036, 498, 0, 30, 0] combine_rows_per_src=[8192, 8138, 8102, 8182, 8188, 412, 5208, 2276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45840 payload_rows_remote=38476 dup_rows=19696 dup_rows_remote=12540 dedup_ratio=0.300537 router_tokens_per_expert=[0, 0, 5, 713, 0, 13, 8, 1, 0, 75, 3144, 0, 0, 0, 43, 0, 0, 484, 0, 4029, 2120, 0, 1423, 4, 37, 135, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 2, 0, 7, 0, 0, 0, 0, 0, 4009, 0, 0, 2014, 0, 9, 0, 9, 0, 0, 3286, 0, 2948, 16, 4080, 0] payload_rows_per_dst=[1442, 6506, 8192, 344, 6172, 7656, 8164, 7364] dup_rows_per_dst=[38, 18, 7928, 0, 636, 2, 3918, 7156] combine_rows_per_src=[1442, 6506, 8192, 344, 6172, 7656, 8164, 7364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54894 payload_rows_remote=46702 dup_rows=10642 dup_rows_remote=10370 dedup_ratio=0.162384 router_tokens_per_expert=[0, 0, 0, 4037, 0, 0, 0, 0, 26, 0, 4064, 1, 90, 0, 51, 0, 0, 0, 0, 342, 0, 5, 4056, 0, 0, 0, 4, 239, 3468, 0, 0, 5, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 1694, 0, 334, 215, 410, 0, 3774, 0, 284, 0, 0, 0, 0, 0, 1745, 0, 3854, 0, 4051, 0] payload_rows_per_dst=[8076, 8192, 8122, 7420, 8124, 3388, 8082, 3490] dup_rows_per_dst=[7706, 272, 684, 12, 16, 1098, 854, 0] combine_rows_per_src=[8076, 8192, 8122, 7420, 8124, 3388, 8082, 3490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45126 payload_rows_remote=41132 dup_rows=20410 dup_rows_remote=20390 dedup_ratio=0.311432 router_tokens_per_expert=[0, 0, 1, 2222, 0, 0, 4096, 0, 885, 0, 3703, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1, 0, 9, 0, 886, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 4032, 0, 0, 0, 305, 0, 55, 4084, 0, 0, 10, 0, 0, 1, 2, 0, 4007, 0, 4066, 143] payload_rows_per_dst=[8192, 7624, 3994, 5036, 8192, 8082, 3714, 292] dup_rows_per_dst=[8512, 1552, 20, 6216, 222, 3666, 222, 0] combine_rows_per_src=[8192, 7624, 3994, 5036, 8192, 8082, 3714, 292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=183672832 dispatch_meta_bytes=230056 combine_payload_bytes=183672832 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183902888 ingress_bytes=208838656 link_bytes=208838656 token_bytes=367575720 weight_movement_bytes=25165824 total_nvlink_bytes=392741544 weight_to_token_ratio=0.068464 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589020872e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=196575232 dispatch_meta_bytes=261720 combine_payload_bytes=196575232 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=196836952 ingress_bytes=246906880 link_bytes=246906880 token_bytes=393412184 weight_movement_bytes=50331648 total_nvlink_bytes=443743832 weight_to_token_ratio=0.127936 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.878676072e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53270 payload_rows_remote=45412 dup_rows=12266 dup_rows_remote=12266 dedup_ratio=0.187164 router_tokens_per_expert=[0, 0, 1, 0, 0, 5, 47, 0, 1416, 0, 3698, 0, 0, 4, 57, 0, 0, 23, 0, 244, 2339, 0, 0, 0, 4092, 2660, 0, 0, 4080, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 359, 0, 0, 0, 96, 0, 25, 3911, 0, 0, 0, 0, 0, 0, 1686, 0, 4096, 0, 3921, 0] payload_rows_per_dst=[104, 8060, 4694, 8184, 7858, 8160, 8018, 8192] dup_rows_per_dst=[2, 2290, 518, 5320, 0, 718, 46, 3372] combine_rows_per_src=[104, 8060, 4694, 8184, 7858, 8160, 8018, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=165912576 dispatch_meta_bytes=199064 combine_payload_bytes=165912576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166111640 ingress_bytes=191078400 link_bytes=191078400 token_bytes=332024216 weight_movement_bytes=25165824 total_nvlink_bytes=357190040 weight_to_token_ratio=0.075795 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.453885845e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=157597696 dispatch_meta_bytes=204064 combine_payload_bytes=157597696 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=157801760 ingress_bytes=258260992 link_bytes=258260992 token_bytes=315399456 weight_movement_bytes=100663296 total_nvlink_bytes=416062752 weight_to_token_ratio=0.319161 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.965067745e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=191291392 dispatch_meta_bytes=228288 combine_payload_bytes=191291392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=191519680 ingress_bytes=216457216 link_bytes=216457216 token_bytes=382811072 weight_movement_bytes=25165824 total_nvlink_bytes=407976896 weight_to_token_ratio=0.065740 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.646989311e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55456 payload_rows_remote=47376 dup_rows=10080 dup_rows_remote=9924 dedup_ratio=0.153809 router_tokens_per_expert=[0, 0, 3931, 149, 30, 2, 244, 0, 0, 9, 69, 2, 0, 0, 69, 0, 0, 667, 0, 0, 0, 0, 75, 4090, 0, 0, 0, 2, 29, 3855, 3411, 0, 0, 2633, 3, 2, 0, 0, 0, 1798, 0, 0, 18, 0, 0, 0, 100, 0, 1, 4028, 0, 0, 0, 0, 0, 89, 58, 0, 52, 0, 4091, 3260, 1, 0] payload_rows_per_dst=[7922, 8192, 7062, 8058, 8182, 1406, 8080, 6554] dup_rows_per_dst=[302, 288, 1334, 6536, 690, 98, 156, 676] combine_rows_per_src=[7922, 8192, 7062, 8058, 8182, 1406, 8080, 6554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=168476672 dispatch_meta_bytes=246088 combine_payload_bytes=168476672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=168722760 ingress_bytes=193642496 link_bytes=193642496 token_bytes=337199432 weight_movement_bytes=25165824 total_nvlink_bytes=362365256 weight_to_token_ratio=0.074632 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.473395653e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=186007552 dispatch_meta_bytes=230712 combine_payload_bytes=186007552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186238264 ingress_bytes=211173376 link_bytes=211173376 token_bytes=372245816 weight_movement_bytes=25165824 total_nvlink_bytes=397411640 weight_to_token_ratio=0.067605 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.606785394e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=194052096 dispatch_meta_bytes=229200 combine_payload_bytes=194052096 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=194281296 ingress_bytes=269549568 link_bytes=269549568 token_bytes=388333392 weight_movement_bytes=75497472 total_nvlink_bytes=463830864 weight_to_token_ratio=0.194414 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=9856 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.050960765e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51390 payload_rows_remote=47562 dup_rows=14146 dup_rows_remote=14136 dedup_ratio=0.215851 router_tokens_per_expert=[0, 3, 141, 0, 0, 0, 4039, 0, 93, 0, 4056, 0, 3757, 0, 10, 0, 0, 1, 0, 38, 4030, 6, 0, 0, 0, 0, 0, 4, 2187, 12, 3885, 2517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 5, 0, 0, 1914, 0, 0, 1, 0, 0, 0, 0, 2, 59, 3, 0, 0, 0, 1529, 0, 765] payload_rows_per_dst=[8166, 8190, 8096, 7850, 0, 3828, 7424, 7836] dup_rows_per_dst=[200, 3148, 4548, 4326, 0, 10, 4, 1910] combine_rows_per_src=[8166, 8190, 8096, 7850, 0, 3828, 7424, 7836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48306 payload_rows_remote=44128 dup_rows=17230 dup_rows_remote=17132 dedup_ratio=0.262909 router_tokens_per_expert=[0, 62, 0, 0, 353, 0, 4096, 0, 0, 0, 3363, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 6, 0, 0, 50, 0, 2082, 2361, 305, 1, 0, 481, 0, 0, 17, 0, 0, 0, 0, 4088, 0, 3136, 4, 15, 2, 0, 0, 2525, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 1829] payload_rows_per_dst=[8192, 8192, 7786, 4178, 6654, 38, 8182, 5084] dup_rows_per_dst=[830, 6726, 0, 98, 1226, 0, 5050, 3300] combine_rows_per_src=[8192, 8192, 7786, 4178, 6654, 38, 8182, 5084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38920 dup_rows=21122 dup_rows_remote=20360 dedup_ratio=0.322296 router_tokens_per_expert=[36, 0, 0, 8, 0, 165, 2402, 563, 340, 0, 1224, 0, 1523, 0, 41, 0, 0, 0, 0, 0, 3409, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 60, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 2429, 211, 0, 15, 3920, 4020, 0, 0, 0, 0, 4, 779, 8, 9, 0, 0, 0, 113, 0, 3536, 0, 4096] payload_rows_per_dst=[5722, 5494, 6822, 7686, 408, 8070, 2020, 8192] dup_rows_per_dst=[764, 762, 6, 0, 0, 12170, 2, 7418] combine_rows_per_src=[5722, 5494, 6822, 7686, 408, 8070, 2020, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43224 payload_rows_remote=35160 dup_rows=22312 dup_rows_remote=21996 dedup_ratio=0.340454 router_tokens_per_expert=[0, 3, 164, 0, 0, 0, 4018, 5, 387, 0, 3218, 0, 4088, 0, 0, 0, 416, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4080, 4044, 3100, 0, 0, 6, 0, 0, 0, 716, 0, 0, 20, 142, 162, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8064, 8192, 8190, 8188, 8192, 364, 320, 1714] dup_rows_per_dst=[316, 7194, 808, 6350, 7644, 0, 0, 0] combine_rows_per_src=[8064, 8192, 8190, 8188, 8192, 364, 320, 1714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47780 payload_rows_remote=41162 dup_rows=17756 dup_rows_remote=17756 dedup_ratio=0.270935 router_tokens_per_expert=[0, 0, 0, 76, 2988, 0, 4004, 0, 0, 99, 0, 0, 4096, 0, 0, 0, 54, 2, 0, 0, 2144, 1, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3309, 8, 1, 0, 0, 17, 0, 0, 2210, 17, 4068, 3, 3952, 0, 818, 3209, 49, 0, 0, 0, 0, 0, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8074, 1406, 8192, 1096, 6012, 8190, 8192, 6618] dup_rows_per_dst=[3764, 0, 4402, 2, 96, 6404, 3088, 0] combine_rows_per_src=[8074, 1406, 8192, 1096, 6012, 8190, 8192, 6618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=194813952 dispatch_meta_bytes=246792 combine_payload_bytes=194813952 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=195060744 ingress_bytes=320643072 link_bytes=320643072 token_bytes=389874696 weight_movement_bytes=125829120 total_nvlink_bytes=515703816 weight_to_token_ratio=0.322742 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=14208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.439723295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=180748288 dispatch_meta_bytes=245040 combine_payload_bytes=180748288 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=180993328 ingress_bytes=231079936 link_bytes=231079936 token_bytes=361741616 weight_movement_bytes=50331648 total_nvlink_bytes=412073264 weight_to_token_ratio=0.139137 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22144 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.758251315e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49862 payload_rows_remote=48112 dup_rows=15674 dup_rows_remote=15324 dedup_ratio=0.239166 router_tokens_per_expert=[0, 0, 2231, 0, 0, 0, 4091, 18, 1, 0, 15, 0, 4005, 0, 0, 0, 100, 0, 0, 319, 631, 0, 0, 0, 0, 0, 0, 9, 1, 0, 5, 4088, 1303, 1, 0, 6, 0, 0, 0, 1, 0, 0, 0, 3917, 0, 3827, 4057, 3649, 0, 0, 0, 0, 0, 0, 0, 5, 397, 0, 0, 0, 0, 91, 0, 0] payload_rows_per_dst=[8192, 8042, 1750, 22, 7662, 8184, 7834, 8176] dup_rows_per_dst=[4488, 0, 350, 8, 2614, 7228, 10, 976] combine_rows_per_src=[8192, 8042, 1750, 22, 7662, 8184, 7834, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46144 payload_rows_remote=37952 dup_rows=19392 dup_rows_remote=11292 dedup_ratio=0.295898 router_tokens_per_expert=[0, 0, 3, 1, 0, 0, 279, 3, 1, 0, 1, 0, 4093, 0, 0, 0, 2, 0, 0, 3, 31, 0, 1, 0, 0, 0, 2082, 34, 350, 0, 4062, 3007, 0, 4004, 0, 19, 0, 0, 0, 27, 0, 0, 1, 4086, 112, 4096, 194, 3135, 0, 0, 0, 0, 0, 0, 0, 3058, 0, 0, 0, 0, 83, 0, 0, 0] payload_rows_per_dst=[566, 8190, 72, 8180, 8192, 6718, 8172, 6054] dup_rows_per_dst=[6, 0, 2, 4876, 8100, 166, 6116, 126] combine_rows_per_src=[566, 8190, 72, 8180, 8192, 6718, 8172, 6054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=168599552 dispatch_meta_bytes=235672 combine_payload_bytes=168599552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=168835224 ingress_bytes=193765376 link_bytes=193765376 token_bytes=337434776 weight_movement_bytes=25165824 total_nvlink_bytes=362600600 weight_to_token_ratio=0.074580 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=20352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.474330628e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=237120 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159653440 ingress_bytes=209747968 link_bytes=209747968 token_bytes=319069760 weight_movement_bytes=50331648 total_nvlink_bytes=369401408 weight_to_token_ratio=0.157745 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595939686e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=144015360 dispatch_meta_bytes=228624 combine_payload_bytes=144015360 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=144243984 ingress_bytes=219512832 link_bytes=219512832 token_bytes=288259344 weight_movement_bytes=75497472 total_nvlink_bytes=363756816 weight_to_token_ratio=0.261908 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.670239018e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57992 payload_rows_remote=49868 dup_rows=7544 dup_rows_remote=7390 dedup_ratio=0.115112 router_tokens_per_expert=[0, 0, 0, 3389, 3985, 4, 4, 1, 0, 3488, 14, 0, 0, 4096, 0, 894, 125, 3887, 0, 0, 773, 2, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2625, 81, 0, 1, 11, 0, 24, 4069, 1012, 16, 4, 25, 0, 0, 0, 0, 0, 240, 0, 0, 12, 0, 0, 0, 2, 0, 0, 0, 1284, 0, 0, 0] payload_rows_per_dst=[6782, 8184, 5850, 8192, 8192, 7234, 8124, 5434] dup_rows_per_dst=[14, 5620, 1350, 0, 180, 130, 154, 96] combine_rows_per_src=[6782, 8184, 5850, 8192, 8192, 7234, 8124, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=197066752 dispatch_meta_bytes=253744 combine_payload_bytes=197066752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197320496 ingress_bytes=222232576 link_bytes=222232576 token_bytes=394387248 weight_movement_bytes=25165824 total_nvlink_bytes=419553072 weight_to_token_ratio=0.063810 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.690933128e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=155451392 dispatch_meta_bytes=196976 combine_payload_bytes=155451392 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=155648368 ingress_bytes=205783040 link_bytes=205783040 token_bytes=311099760 weight_movement_bytes=50331648 total_nvlink_bytes=361431408 weight_to_token_ratio=0.161786 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.565771165e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55338 payload_rows_remote=47146 dup_rows=10198 dup_rows_remote=1560 dedup_ratio=0.155609 router_tokens_per_expert=[3788, 0, 314, 2947, 6, 0, 4082, 0, 0, 2, 0, 631, 0, 0, 0, 0, 0, 26, 8, 0, 0, 0, 0, 0, 0, 1374, 3, 4, 0, 4, 4004, 0, 0, 0, 0, 229, 0, 0, 0, 0, 72, 1274, 1423, 4057, 0, 0, 211, 4061, 28, 0, 0, 0, 0, 0, 0, 12, 4042, 2, 0, 123, 0, 0, 0, 41] payload_rows_per_dst=[6528, 8118, 7632, 8026, 458, 8192, 8192, 8192] dup_rows_per_dst=[6, 1262, 12, 4, 0, 8638, 52, 224] combine_rows_per_src=[6528, 8118, 7632, 8026, 458, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=204259328 dispatch_meta_bytes=229032 combine_payload_bytes=204259328 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=204488360 ingress_bytes=229425152 link_bytes=229425152 token_bytes=408747688 weight_movement_bytes=25165824 total_nvlink_bytes=433913512 weight_to_token_ratio=0.061568 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.745660321e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56932 payload_rows_remote=48798 dup_rows=8604 dup_rows_remote=8604 dedup_ratio=0.131287 router_tokens_per_expert=[4095, 4061, 0, 2492, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 0, 0, 0, 0, 0, 0, 3857, 4096, 0, 0, 23, 266, 0, 0, 1548, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7820, 8134, 42, 8168, 8192, 8192] dup_rows_per_dst=[5284, 98, 0, 0, 0, 124, 3096, 2] combine_rows_per_src=[8192, 8192, 7820, 8134, 42, 8168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51076 payload_rows_remote=43008 dup_rows=14460 dup_rows_remote=11586 dedup_ratio=0.220642 router_tokens_per_expert=[3988, 0, 0, 4081, 0, 0, 3630, 0, 0, 0, 0, 10, 0, 0, 17, 0, 0, 3368, 0, 821, 1, 0, 0, 628, 0, 0, 0, 0, 0, 0, 3426, 0, 0, 0, 41, 2, 11, 0, 4015, 0, 3, 0, 4058, 96, 0, 0, 0, 7, 1, 0, 0, 0, 35, 0, 0, 120, 3508, 268, 36, 171, 0, 162, 264, 0] payload_rows_per_dst=[4112, 240, 7960, 6852, 8192, 8124, 7528, 8068] dup_rows_per_dst=[0, 6, 1676, 1926, 7922, 12, 44, 2874] combine_rows_per_src=[4112, 240, 7960, 6852, 8192, 8124, 7528, 8068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47156 payload_rows_remote=38964 dup_rows=18380 dup_rows_remote=10664 dedup_ratio=0.280457 router_tokens_per_expert=[0, 1931, 30, 0, 0, 0, 3829, 2, 346, 0, 355, 0, 2042, 33, 0, 14, 0, 0, 0, 0, 0, 0, 0, 2913, 73, 3958, 5, 183, 0, 866, 4073, 0, 0, 376, 0, 2789, 0, 275, 0, 4, 0, 1, 1107, 4057, 0, 0, 0, 0, 0, 0, 0, 242, 0, 15, 0, 0, 0, 0, 0, 0, 3152, 0, 0, 97] payload_rows_per_dst=[3926, 8192, 5826, 8186, 6400, 2, 8128, 6496] dup_rows_per_dst=[0, 7716, 0, 10130, 488, 0, 44, 2] combine_rows_per_src=[3926, 8192, 5826, 8186, 6400, 2, 8128, 6496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43872 payload_rows_remote=35680 dup_rows=21664 dup_rows_remote=6944 dedup_ratio=0.330566 router_tokens_per_expert=[4078, 4096, 504, 2687, 0, 0, 4089, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 0, 0, 0, 0, 69, 0, 0, 0, 10, 668, 0, 0, 0, 0, 1, 0, 0, 2020, 0, 0, 0, 4092, 2575, 0, 859, 2, 4026, 0, 0, 44, 0, 4, 0, 0, 0, 1459, 0, 0, 4, 5, 0, 1431, 10] payload_rows_per_dst=[8192, 8190, 24, 1492, 4042, 8076, 8084, 5772] dup_rows_per_dst=[14720, 5190, 0, 2, 0, 1698, 8, 46] combine_rows_per_src=[8192, 8190, 24, 1492, 4042, 8076, 8084, 5772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=193110016 dispatch_meta_bytes=194824 combine_payload_bytes=193110016 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=193304840 ingress_bytes=268607488 link_bytes=268607488 token_bytes=386414856 weight_movement_bytes=75497472 total_nvlink_bytes=461912328 weight_to_token_ratio=0.195379 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=12032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.043792625e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51352 payload_rows_remote=43486 dup_rows=14184 dup_rows_remote=14180 dedup_ratio=0.216431 router_tokens_per_expert=[3890, 1640, 9, 338, 0, 63, 2813, 0, 0, 0, 1, 35, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 3747, 0, 3413, 0, 0, 0, 11, 796, 0, 3, 0, 0, 4082, 0, 0, 4025, 0, 0, 0, 2819, 3659, 0, 0, 0, 509, 0, 0, 3, 0, 16, 149, 1, 0, 0, 0, 5, 2, 1, 325, 217, 8] payload_rows_per_dst=[8192, 7554, 7866, 7204, 8164, 5354, 5930, 1088] dup_rows_per_dst=[3688, 150, 4, 1236, 8056, 988, 34, 28] combine_rows_per_src=[8192, 7554, 7866, 7204, 8164, 5354, 5930, 1088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=199876608 dispatch_meta_bytes=229608 combine_payload_bytes=199876608 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=200106216 ingress_bytes=250208256 link_bytes=250208256 token_bytes=399982824 weight_movement_bytes=50331648 total_nvlink_bytes=450314472 weight_to_token_ratio=0.125835 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.903795728e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44830 payload_rows_remote=36652 dup_rows=20706 dup_rows_remote=20684 dedup_ratio=0.315948 router_tokens_per_expert=[4054, 77, 25, 337, 0, 2, 4080, 0, 0, 0, 49, 1137, 0, 0, 0, 0, 0, 270, 0, 0, 1, 0, 0, 3980, 0, 3793, 0, 0, 0, 71, 2790, 0, 33, 0, 0, 9, 0, 0, 4057, 1, 0, 0, 25, 31, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 3877, 0, 0, 3782, 285, 0, 0, 0] payload_rows_per_dst=[828, 2312, 8150, 7690, 8178, 1316, 8164, 8192] dup_rows_per_dst=[54, 122, 8460, 4324, 22, 28, 0, 7696] combine_rows_per_src=[828, 2312, 8150, 7690, 8178, 1316, 8164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=176160768 dispatch_meta_bytes=218376 combine_payload_bytes=176160768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=176379144 ingress_bytes=201326592 link_bytes=201326592 token_bytes=352539912 weight_movement_bytes=25165824 total_nvlink_bytes=377705736 weight_to_token_ratio=0.071384 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.531862745e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=198512 combine_payload_bytes=159596544 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=159795056 ingress_bytes=209928192 link_bytes=209928192 token_bytes=319391600 weight_movement_bytes=50331648 total_nvlink_bytes=369723248 weight_to_token_ratio=0.157586 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597310982e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=146145280 dispatch_meta_bytes=170496 combine_payload_bytes=146145280 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=146315776 ingress_bytes=271974400 link_bytes=271974400 token_bytes=292461056 weight_movement_bytes=125829120 total_nvlink_bytes=418290176 weight_to_token_ratio=0.430242 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.069410935e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48760 payload_rows_remote=40576 dup_rows=16776 dup_rows_remote=9730 dedup_ratio=0.255981 router_tokens_per_expert=[0, 0, 1534, 0, 3759, 0, 0, 0, 2771, 3785, 1744, 0, 4007, 4038, 0, 4092, 12, 0, 0, 0, 0, 899, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 13, 0, 0, 3, 0, 0, 24, 0, 0, 0, 0, 10, 205, 82, 2256, 0, 3516, 0, 1, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5236, 7662, 1802, 5308, 7588, 4902, 8184, 8078] dup_rows_per_dst=[62, 5016, 20, 0, 62, 204, 7046, 4366] combine_rows_per_src=[5236, 7662, 1802, 5308, 7588, 4902, 8184, 8078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=178118656 dispatch_meta_bytes=230664 combine_payload_bytes=178118656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=178349320 ingress_bytes=203284480 link_bytes=203284480 token_bytes=356467976 weight_movement_bytes=25165824 total_nvlink_bytes=381633800 weight_to_token_ratio=0.070598 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=14976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.546760011e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=150126592 dispatch_meta_bytes=229344 combine_payload_bytes=150126592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=150355936 ingress_bytes=175292416 link_bytes=175292416 token_bytes=300482528 weight_movement_bytes=25165824 total_nvlink_bytes=325648352 weight_to_token_ratio=0.083751 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.333772747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=166199296 dispatch_meta_bytes=201224 combine_payload_bytes=166199296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166400520 ingress_bytes=191365120 link_bytes=191365120 token_bytes=332599816 weight_movement_bytes=25165824 total_nvlink_bytes=357765640 weight_to_token_ratio=0.075664 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.456067453e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53592 payload_rows_remote=46538 dup_rows=11944 dup_rows_remote=6562 dedup_ratio=0.182251 router_tokens_per_expert=[2, 0, 0, 36, 3815, 4017, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 0, 757, 0, 149, 0, 0, 2364, 4096, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3976, 0, 14, 0, 0, 3401, 2673, 48, 0, 7, 0, 0, 89, 10, 1, 0, 0, 0, 0, 0, 698, 312, 0, 0, 2, 2, 2592, 0, 0] payload_rows_per_dst=[8110, 0, 6342, 8192, 8000, 7054, 7702, 8192] dup_rows_per_dst=[0, 0, 286, 0, 0, 5382, 1022, 5254] combine_rows_per_src=[8110, 0, 6342, 8192, 8000, 7054, 7702, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46764 payload_rows_remote=45522 dup_rows=18772 dup_rows_remote=18772 dedup_ratio=0.286438 router_tokens_per_expert=[0, 0, 0, 0, 4060, 3538, 838, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 159, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 3, 0, 3860, 0, 0, 0, 0, 2755, 224, 0, 0, 0, 0, 61, 2, 0, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 2762] payload_rows_per_dst=[8054, 8192, 3190, 1242, 8130, 5806, 3990, 8160] dup_rows_per_dst=[698, 5320, 0, 0, 6950, 278, 0, 5526] combine_rows_per_src=[8054, 8192, 3190, 1242, 8130, 5806, 3990, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48140 payload_rows_remote=39948 dup_rows=17396 dup_rows_remote=16590 dedup_ratio=0.265442 router_tokens_per_expert=[0, 0, 0, 0, 4096, 4094, 24, 1, 0, 9, 6, 0, 3791, 0, 0, 0, 4065, 0, 19, 3, 0, 0, 3552, 1012, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 2860, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 914, 175, 0, 3062, 79, 0, 0, 463, 0, 0, 31, 237, 0, 0, 156, 0, 5, 5, 0] payload_rows_per_dst=[8188, 30, 2024, 7602, 5720, 8192, 8192, 8192] dup_rows_per_dst=[50, 0, 624, 7104, 0, 2152, 6660, 806] combine_rows_per_src=[8188, 30, 2024, 7602, 5720, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52114 payload_rows_remote=43980 dup_rows=13422 dup_rows_remote=7602 dedup_ratio=0.204803 router_tokens_per_expert=[0, 0, 0, 0, 3242, 48, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 2967, 0, 0, 166, 0, 6, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 772, 0, 4039, 47, 0, 13, 0, 0, 3886, 63, 0, 0, 0, 0, 1986, 3333, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[6580, 8134, 6378, 8192, 8172, 7978, 6666, 14] dup_rows_per_dst=[0, 5820, 98, 0, 3612, 3892, 0, 0] combine_rows_per_src=[6580, 8134, 6378, 8192, 8172, 7978, 6666, 14] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=36760 payload_rows_remote=28568 dup_rows=28776 dup_rows_remote=13156 dedup_ratio=0.439087 router_tokens_per_expert=[0, 0, 0, 1489, 4096, 191, 3917, 0, 0, 0, 3, 0, 4095, 0, 0, 2727, 3668, 0, 4067, 51, 0, 0, 4069, 0, 87, 0, 0, 0, 70, 0, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 3781, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 3, 0, 414] payload_rows_per_dst=[8192, 8192, 8182, 8192, 100, 3052, 0, 850] dup_rows_per_dst=[15620, 12794, 102, 260, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8182, 8192, 100, 3052, 0, 850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=190619648 dispatch_meta_bytes=212400 combine_payload_bytes=190619648 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=190832048 ingress_bytes=240951296 link_bytes=240951296 token_bytes=381451696 weight_movement_bytes=50331648 total_nvlink_bytes=431783344 weight_to_token_ratio=0.131948 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.833360959e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=186458112 dispatch_meta_bytes=257176 combine_payload_bytes=186458112 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=186715288 ingress_bytes=236789760 link_bytes=236789760 token_bytes=373173400 weight_movement_bytes=50331648 total_nvlink_bytes=423505048 weight_to_token_ratio=0.134875 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.801696478e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51862 payload_rows_remote=45018 dup_rows=13674 dup_rows_remote=10026 dedup_ratio=0.208649 router_tokens_per_expert=[0, 0, 0, 131, 3654, 1863, 666, 0, 0, 0, 1, 0, 4059, 0, 0, 41, 1100, 0, 3271, 20, 0, 1915, 3994, 40, 0, 0, 0, 0, 949, 0, 0, 85, 0, 0, 0, 3738, 0, 0, 0, 0, 2596, 63, 0, 115, 0, 0, 0, 0, 452, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 15, 3, 0, 3986] payload_rows_per_dst=[6738, 8136, 6844, 8192, 7476, 5530, 910, 8036] dup_rows_per_dst=[2282, 2266, 3648, 1864, 0, 18, 0, 3596] combine_rows_per_src=[6738, 8136, 6844, 8192, 7476, 5530, 910, 8036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43634 payload_rows_remote=35816 dup_rows=21902 dup_rows_remote=21902 dedup_ratio=0.334198 router_tokens_per_expert=[0, 0, 0, 0, 4088, 0, 1, 0, 0, 0, 845, 0, 4095, 0, 0, 2088, 4016, 0, 3190, 4, 0, 2, 3491, 2587, 0, 0, 0, 0, 25, 2, 0, 0, 0, 0, 0, 3909, 0, 0, 0, 0, 47, 0, 0, 2, 0, 0, 0, 0, 4067, 1, 0, 0, 3, 0, 0, 39, 0, 0, 0, 0, 7, 202, 0, 57] payload_rows_per_dst=[2, 4200, 8156, 7036, 7818, 94, 8136, 8192] dup_rows_per_dst=[0, 2452, 11442, 0, 0, 4, 7488, 516] combine_rows_per_src=[2, 4200, 8156, 7036, 7818, 94, 8136, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=163627008 dispatch_meta_bytes=226152 combine_payload_bytes=163627008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=163853160 ingress_bytes=188792832 link_bytes=188792832 token_bytes=327480168 weight_movement_bytes=25165824 total_nvlink_bytes=352645992 weight_to_token_ratio=0.076847 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.436495313e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=180142080 dispatch_meta_bytes=206328 combine_payload_bytes=180142080 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=180348408 ingress_bytes=230473728 link_bytes=230473728 token_bytes=360490488 weight_movement_bytes=50331648 total_nvlink_bytes=410822136 weight_to_token_ratio=0.139620 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.753638772e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=117014528 dispatch_meta_bytes=166896 combine_payload_bytes=117014528 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=117181424 ingress_bytes=167346176 link_bytes=167346176 token_bytes=234195952 weight_movement_bytes=50331648 total_nvlink_bytes=284527600 weight_to_token_ratio=0.214913 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=14336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.273311042e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55212 payload_rows_remote=50374 dup_rows=10324 dup_rows_remote=9530 dedup_ratio=0.157532 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 1282, 0, 3058, 0, 4091, 0, 0, 37, 0, 0, 0, 0, 0, 0, 3507, 0, 1571, 0, 3941, 7, 0, 0, 4044, 0, 0, 23, 0, 0, 264, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2385, 0, 0, 426, 5, 0, 0, 0, 35, 2, 4042, 0, 4034, 0] payload_rows_per_dst=[2568, 8192, 7360, 7886, 8192, 8088, 4838, 8088] dup_rows_per_dst=[0, 6180, 2796, 56, 428, 0, 794, 70] combine_rows_per_src=[2568, 8192, 7360, 7886, 8192, 8088, 4838, 8088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=184393728 dispatch_meta_bytes=220176 combine_payload_bytes=184393728 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=184613904 ingress_bytes=285057024 link_bytes=285057024 token_bytes=369007632 weight_movement_bytes=100663296 total_nvlink_bytes=469670928 weight_to_token_ratio=0.272795 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=11136 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.168954587e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=146702336 dispatch_meta_bytes=230872 combine_payload_bytes=146702336 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=146933208 ingress_bytes=197033984 link_bytes=197033984 token_bytes=293635544 weight_movement_bytes=50331648 total_nvlink_bytes=343967192 weight_to_token_ratio=0.171409 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23424 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.499200958e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50424 payload_rows_remote=48130 dup_rows=15112 dup_rows_remote=14332 dedup_ratio=0.230591 router_tokens_per_expert=[0, 0, 0, 0, 4069, 3511, 0, 103, 14, 6, 0, 219, 0, 249, 0, 14, 1161, 3918, 0, 2667, 0, 0, 0, 1, 0, 0, 4000, 0, 0, 0, 0, 0, 4086, 1, 6, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 63, 2, 0, 14, 111, 23, 3186, 0, 4056, 0, 0, 0, 0, 0, 12, 59, 0, 8] payload_rows_per_dst=[7228, 964, 7420, 8000, 8182, 2294, 8182, 8154] dup_rows_per_dst=[0, 40, 7300, 0, 252, 780, 6598, 142] combine_rows_per_src=[7228, 964, 7420, 8000, 8182, 2294, 8182, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=206331904 dispatch_meta_bytes=239616 combine_payload_bytes=206331904 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=206571520 ingress_bytes=281829376 link_bytes=281829376 token_bytes=412903424 weight_movement_bytes=75497472 total_nvlink_bytes=488400896 weight_to_token_ratio=0.182845 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=11520 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.144395916e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52044 payload_rows_remote=43854 dup_rows=13492 dup_rows_remote=13490 dedup_ratio=0.205872 router_tokens_per_expert=[6, 2969, 1065, 0, 4096, 3791, 0, 0, 46, 0, 0, 83, 0, 12, 0, 0, 0, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 4085, 60, 4022, 0, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[8078, 3088, 7284, 8190, 8192, 932, 8088, 8192] dup_rows_per_dst=[7584, 260, 0, 2, 0, 458, 5180, 8] combine_rows_per_src=[8078, 3088, 7284, 8190, 8192, 932, 8088, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=197140480 dispatch_meta_bytes=249848 combine_payload_bytes=197140480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197390328 ingress_bytes=222306304 link_bytes=222306304 token_bytes=394530808 weight_movement_bytes=25165824 total_nvlink_bytes=419696632 weight_to_token_ratio=0.063787 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=13056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.691494113e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48548 payload_rows_remote=40420 dup_rows=16988 dup_rows_remote=16958 dedup_ratio=0.259216 router_tokens_per_expert=[13, 0, 0, 0, 4008, 4053, 0, 16, 4091, 210, 0, 0, 51, 0, 1, 0, 167, 2565, 0, 0, 0, 31, 0, 0, 0, 0, 3061, 0, 0, 9, 0, 59, 3608, 1, 42, 0, 0, 0, 2, 0, 20, 0, 0, 219, 983, 3877, 128, 10, 0, 588, 0, 33, 3928, 582, 341, 0, 5, 2, 0, 0, 0, 49, 15, 0] payload_rows_per_dst=[5614, 8182, 188, 3050, 7266, 7946, 8174, 8128] dup_rows_per_dst=[40, 524, 48, 2, 5756, 7658, 2930, 30] combine_rows_per_src=[5614, 8182, 188, 3050, 7266, 7946, 8174, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55358 payload_rows_remote=47268 dup_rows=10178 dup_rows_remote=9384 dedup_ratio=0.155304 router_tokens_per_expert=[10, 0, 28, 0, 2878, 4091, 3, 0, 0, 412, 0, 0, 25, 10, 0, 0, 4094, 892, 0, 0, 0, 0, 390, 0, 0, 13, 3493, 0, 0, 0, 0, 3, 291, 317, 0, 649, 0, 99, 4096, 0, 0, 0, 0, 0, 0, 3950, 74, 0, 0, 0, 0, 0, 3995, 559, 0, 0, 0, 86, 0, 9, 0, 2236, 65, 0] payload_rows_per_dst=[8190, 8090, 8192, 7012, 8192, 7918, 1118, 6646] dup_rows_per_dst=[74, 794, 2560, 6, 2712, 130, 0, 3902] combine_rows_per_src=[8190, 8090, 8192, 7012, 8192, 7918, 1118, 6646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=179625984 dispatch_meta_bytes=229376 combine_payload_bytes=179625984 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=179855360 ingress_bytes=229957632 link_bytes=229957632 token_bytes=359481344 weight_movement_bytes=50331648 total_nvlink_bytes=409812992 weight_to_token_ratio=0.140012 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=28032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.749711878e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55264 payload_rows_remote=47072 dup_rows=10272 dup_rows_remote=7210 dedup_ratio=0.156738 router_tokens_per_expert=[0, 0, 2, 0, 4087, 4093, 0, 0, 29, 53, 0, 0, 0, 0, 0, 0, 318, 3738, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 15, 0, 0, 0, 3, 0, 4057, 0, 0, 0, 0, 3533, 0, 0, 0, 3, 0, 0, 2947, 1686, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8192, 5394, 8092, 8192, 8180, 8114, 3378, 5722] dup_rows_per_dst=[3062, 58, 56, 0, 30, 7066, 0, 0] combine_rows_per_src=[8192, 5394, 8092, 8192, 8180, 8114, 3378, 5722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56362 payload_rows_remote=48170 dup_rows=9174 dup_rows_remote=9172 dedup_ratio=0.139984 router_tokens_per_expert=[0, 0, 2, 0, 4096, 4095, 0, 0, 96, 4, 0, 0, 0, 4044, 0, 0, 4090, 1049, 0, 2, 0, 0, 0, 0, 0, 3414, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 795, 568, 398, 0, 0, 15, 0, 0, 167, 31, 1714, 0, 0, 0, 0, 0, 5, 9, 0, 0] payload_rows_per_dst=[8190, 8192, 8182, 8178, 8192, 3480, 3756, 8192] dup_rows_per_dst=[4, 96, 2100, 6804, 2, 42, 98, 28] combine_rows_per_src=[8190, 8192, 8182, 8178, 8192, 3480, 3756, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53058 payload_rows_remote=44868 dup_rows=12478 dup_rows_remote=9908 dedup_ratio=0.190399 router_tokens_per_expert=[0, 0, 3, 0, 4079, 2181, 0, 0, 704, 1185, 0, 0, 0, 0, 0, 0, 4093, 943, 0, 344, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3422, 0, 3690, 0, 0, 4, 0, 0, 4086, 4023, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0] payload_rows_per_dst=[4364, 8192, 8190, 7906, 72, 8122, 8054, 8158] dup_rows_per_dst=[4, 3758, 2570, 0, 2, 6102, 0, 42] combine_rows_per_src=[4364, 8192, 8190, 7906, 72, 8122, 8054, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=165560320 dispatch_meta_bytes=229512 combine_payload_bytes=165560320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=165789832 ingress_bytes=190726144 link_bytes=190726144 token_bytes=331350152 weight_movement_bytes=25165824 total_nvlink_bytes=356515976 weight_to_token_ratio=0.075949 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=30208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.451205584e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=193609728 dispatch_meta_bytes=226608 combine_payload_bytes=193609728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193836336 ingress_bytes=218775552 link_bytes=218775552 token_bytes=387446064 weight_movement_bytes=25165824 total_nvlink_bytes=412611888 weight_to_token_ratio=0.064953 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.664629170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50370 payload_rows_remote=44562 dup_rows=15166 dup_rows_remote=11852 dedup_ratio=0.231415 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3163, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1729, 12, 0, 529, 0, 4068, 4042, 3, 1260, 0, 0, 0, 212, 27, 0, 0, 451, 0, 1626, 0, 2866, 0, 4096, 2, 0, 0, 435, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 24] payload_rows_per_dst=[8192, 0, 8192, 8192, 7940, 3962, 5808, 8084] dup_rows_per_dst=[0, 0, 6320, 4484, 796, 204, 3314, 48] combine_rows_per_src=[8192, 0, 8192, 8192, 7940, 3962, 5808, 8084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=192806912 dispatch_meta_bytes=217128 combine_payload_bytes=192806912 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=193024040 ingress_bytes=268304384 link_bytes=268304384 token_bytes=385830952 weight_movement_bytes=75497472 total_nvlink_bytes=461328424 weight_to_token_ratio=0.195675 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.041486354e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=197304320 dispatch_meta_bytes=229368 combine_payload_bytes=197304320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197533688 ingress_bytes=222470144 link_bytes=222470144 token_bytes=394838008 weight_movement_bytes=25165824 total_nvlink_bytes=420003832 weight_to_token_ratio=0.063737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=22144 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.692740746e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=183779328 dispatch_meta_bytes=219104 combine_payload_bytes=183779328 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=183998432 ingress_bytes=234110976 link_bytes=234110976 token_bytes=367777760 weight_movement_bytes=50331648 total_nvlink_bytes=418109408 weight_to_token_ratio=0.136853 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=14080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781314027e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55130 payload_rows_remote=47348 dup_rows=10406 dup_rows_remote=3280 dedup_ratio=0.158783 router_tokens_per_expert=[0, 1358, 19, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 20, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 8, 0, 3566, 0, 3880, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 61, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 2902, 130, 0, 4096, 0, 0, 0, 1, 208, 0, 0, 4062, 0] payload_rows_per_dst=[2754, 6056, 8192, 7782, 8192, 8028, 5934, 8192] dup_rows_per_dst=[0, 30, 44, 7126, 470, 120, 2266, 350] combine_rows_per_src=[2754, 6056, 8192, 7782, 8192, 8028, 5934, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56274 payload_rows_remote=48082 dup_rows=9262 dup_rows_remote=8218 dedup_ratio=0.141327 router_tokens_per_expert=[0, 0, 0, 0, 0, 4061, 0, 0, 0, 128, 0, 0, 0, 2, 0, 0, 0, 0, 3671, 0, 30, 0, 3970, 67, 1, 97, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 3, 0, 4, 0, 250, 0, 0, 4095, 15, 0, 60, 2, 446, 0, 0, 0, 2728, 0, 0, 3684, 0, 0, 0, 0, 1339, 129, 48, 3849, 0] payload_rows_per_dst=[8122, 256, 8074, 8186, 7850, 8192, 7440, 8154] dup_rows_per_dst=[0, 4, 60, 188, 6, 1044, 5384, 2576] combine_rows_per_src=[8122, 256, 8074, 8186, 7850, 8192, 7440, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=182525952 dispatch_meta_bytes=225656 combine_payload_bytes=182525952 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=182751608 ingress_bytes=232857600 link_bytes=232857600 token_bytes=365277560 weight_movement_bytes=50331648 total_nvlink_bytes=415609208 weight_to_token_ratio=0.137790 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.771777284e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54340 payload_rows_remote=46242 dup_rows=11196 dup_rows_remote=9892 dedup_ratio=0.170837 router_tokens_per_expert=[59, 442, 0, 0, 0, 0, 0, 0, 3832, 0, 29, 3803, 0, 0, 4, 0, 0, 0, 3991, 0, 0, 0, 4080, 0, 0, 0, 1, 35, 1779, 0, 3075, 0, 2, 0, 0, 0, 30, 0, 0, 0, 107, 0, 4041, 0, 267, 17, 0, 0, 0, 0, 0, 30, 0, 133, 2340, 0, 916, 2, 34, 0, 0, 0, 3719, 0] payload_rows_per_dst=[884, 7826, 8160, 8192, 8044, 8190, 4946, 8098] dup_rows_per_dst=[118, 7510, 0, 1588, 2, 674, 0, 1304] combine_rows_per_src=[884, 7826, 8160, 8192, 8044, 8190, 4946, 8098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49844 payload_rows_remote=47116 dup_rows=15692 dup_rows_remote=15518 dedup_ratio=0.239441 router_tokens_per_expert=[0, 2396, 0, 0, 0, 240, 544, 0, 11, 72, 35, 0, 0, 31, 0, 11, 0, 0, 1431, 0, 0, 20, 3831, 0, 0, 0, 0, 0, 694, 0, 4096, 4, 0, 921, 0, 0, 0, 0, 266, 1076, 0, 144, 3135, 2361, 0, 0, 0, 1010, 0, 0, 0, 3558, 0, 4, 3649, 0, 0, 0, 0, 0, 0, 1, 3227, 0] payload_rows_per_dst=[7234, 2728, 7702, 8192, 5368, 6956, 5208, 6456] dup_rows_per_dst=[3842, 174, 0, 6100, 2020, 3556, 0, 0] combine_rows_per_src=[7234, 2728, 7702, 8192, 5368, 6956, 5208, 6456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=193937408 dispatch_meta_bytes=202512 combine_payload_bytes=193937408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=194139920 ingress_bytes=219103232 link_bytes=219103232 token_bytes=388077328 weight_movement_bytes=25165824 total_nvlink_bytes=413243152 weight_to_token_ratio=0.064847 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=19584 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.667122436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=196943872 dispatch_meta_bytes=225200 combine_payload_bytes=196943872 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=197169072 ingress_bytes=272441344 link_bytes=272441344 token_bytes=394112944 weight_movement_bytes=75497472 total_nvlink_bytes=469610416 weight_to_token_ratio=0.191563 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.072963839e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=36712 payload_rows_remote=28520 dup_rows=28824 dup_rows_remote=13422 dedup_ratio=0.439819 router_tokens_per_expert=[189, 3528, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2982, 0, 0, 0, 18, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3842, 3, 0, 0, 0, 215, 0, 0, 0, 3, 0, 0, 4092, 0, 138, 0, 0, 0, 948, 0, 4094, 0] payload_rows_per_dst=[8192, 1028, 8192, 8192, 2482, 434, 0, 8192] dup_rows_per_dst=[15402, 0, 10072, 1122, 0, 60, 0, 2168] combine_rows_per_src=[8192, 1028, 8192, 8192, 2482, 434, 0, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48712 payload_rows_remote=40520 dup_rows=16824 dup_rows_remote=16824 dedup_ratio=0.256714 router_tokens_per_expert=[0, 3580, 0, 0, 0, 0, 0, 0, 614, 0, 0, 4074, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 8, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4056, 724, 0, 0, 0, 3726, 0, 0, 0, 3350, 0, 0, 4091, 0, 42, 0, 0, 0, 0, 1, 4082, 0] payload_rows_per_dst=[7160, 8174, 0, 618, 8192, 8192, 8184, 8192] dup_rows_per_dst=[0, 1232, 0, 6, 0, 8830, 6698, 58] combine_rows_per_src=[7160, 8174, 0, 618, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52920 payload_rows_remote=45958 dup_rows=12616 dup_rows_remote=12616 dedup_ratio=0.192505 router_tokens_per_expert=[1, 3118, 0, 0, 0, 2787, 0, 0, 4, 0, 0, 2918, 364, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 3449, 0, 0, 0, 6, 0, 2, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1471, 1334, 0, 0, 0, 142, 0, 0, 0, 3232, 0, 0, 4092, 0, 578, 0, 0, 0, 0, 0, 1065, 0] payload_rows_per_dst=[6870, 8186, 6962, 8162, 8192, 4800, 6464, 3284] dup_rows_per_dst=[4942, 6570, 0, 2, 0, 1100, 0, 2] combine_rows_per_src=[6870, 8186, 6962, 8162, 8192, 4800, 6464, 3284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=189407232 dispatch_meta_bytes=224536 combine_payload_bytes=189407232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189631768 ingress_bytes=214573056 link_bytes=214573056 token_bytes=379039000 weight_movement_bytes=25165824 total_nvlink_bytes=404204824 weight_to_token_ratio=0.066394 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.632653031e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=192987136 dispatch_meta_bytes=250536 combine_payload_bytes=192987136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193237672 ingress_bytes=218152960 link_bytes=218152960 token_bytes=386224808 weight_movement_bytes=25165824 total_nvlink_bytes=411390632 weight_to_token_ratio=0.065158 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.659891964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6165237776 prefetch_bytes=402653184 reduce_bytes=276824064 weight_movement_bytes=679477248 egress_bytes=3084480528 ingress_bytes=3760234496 link_bytes=3760234496 total_nvlink_bytes=6844715024 weight_to_token_ratio=0.110211 prefetch_share_of_ingress=0.107082 dedup_ratio=0.197086 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=11 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=17399 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.788189988e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5910769352 prefetch_bytes=402653184 reduce_bytes=478150656 weight_movement_bytes=880803840 egress_bytes=2957168328 ingress_bytes=3834404864 link_bytes=3834404864 total_nvlink_bytes=6791573192 weight_to_token_ratio=0.149017 prefetch_share_of_ingress=0.105011 dedup_ratio=0.208065 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=19 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=18796 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.823461913e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50938 payload_rows_remote=43016 dup_rows=14598 dup_rows_remote=14480 dedup_ratio=0.222748 router_tokens_per_expert=[4095, 31, 0, 570, 306, 0, 3841, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 3925, 127, 0, 6, 0, 32, 0, 0, 1, 0, 4085, 0, 0, 1, 6, 0, 0, 0, 4096, 2, 2, 3476, 3, 0, 0, 90, 88, 0, 0, 1, 0, 4085, 0, 247, 0, 0, 0, 0, 0] payload_rows_per_dst=[1814, 8192, 8180, 318, 8174, 8146, 7922, 8192] dup_rows_per_dst=[0, 2, 1926, 12, 5398, 6670, 118, 472] combine_rows_per_src=[1814, 8192, 8180, 318, 8174, 8146, 7922, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=116817920 dispatch_meta_bytes=167768 combine_payload_bytes=116817920 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=116985688 ingress_bytes=141983744 link_bytes=141983744 token_bytes=233803608 weight_movement_bytes=25165824 total_nvlink_bytes=258969432 weight_to_token_ratio=0.107637 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=14464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.080332239e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=165969920 dispatch_meta_bytes=229376 combine_payload_bytes=165969920 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166199296 ingress_bytes=191135744 link_bytes=191135744 token_bytes=332169216 weight_movement_bytes=25165824 total_nvlink_bytes=357335040 weight_to_token_ratio=0.075762 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.454322167e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=188243968 dispatch_meta_bytes=234296 combine_payload_bytes=188243968 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=188478264 ingress_bytes=238575616 link_bytes=238575616 token_bytes=376722232 weight_movement_bytes=50331648 total_nvlink_bytes=427053880 weight_to_token_ratio=0.133604 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=26496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.815284779e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5367611232 prefetch_bytes=377487360 reduce_bytes=201326592 weight_movement_bytes=578813952 egress_bytes=2685525856 ingress_bytes=3260899328 link_bytes=3260899328 total_nvlink_bytes=5946425184 weight_to_token_ratio=0.107835 prefetch_share_of_ingress=0.115762 dedup_ratio=0.256937 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=8 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=18034 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.550729758e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5410023600 prefetch_bytes=427819008 reduce_bytes=276824064 weight_movement_bytes=704643072 egress_bytes=2706786480 ingress_bytes=3407880192 link_bytes=3407880192 total_nvlink_bytes=6114666672 weight_to_token_ratio=0.130248 prefetch_share_of_ingress=0.125538 dedup_ratio=0.262573 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=11 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=18542 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.620626917e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=176193536 dispatch_meta_bytes=229984 combine_payload_bytes=176193536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=176423520 ingress_bytes=201359360 link_bytes=201359360 token_bytes=352617056 weight_movement_bytes=25165824 total_nvlink_bytes=377782880 weight_to_token_ratio=0.071369 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.532112072e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=4936750816 prefetch_bytes=352321536 reduce_bytes=503316480 weight_movement_bytes=855638016 egress_bytes=2470000352 ingress_bytes=3322388480 link_bytes=3322388480 total_nvlink_bytes=5792388832 weight_to_token_ratio=0.173320 prefetch_share_of_ingress=0.106045 dedup_ratio=0.306332 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=20 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=18034 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.579971095e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5787784208 prefetch_bytes=377487360 reduce_bytes=352321536 weight_movement_bytes=729808896 egress_bytes=2895746064 ingress_bytes=3621847040 link_bytes=3621847040 total_nvlink_bytes=6517593104 weight_to_token_ratio=0.126095 prefetch_share_of_ingress=0.104225 dedup_ratio=0.223431 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=14 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=18669 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.722379448e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5829196688 prefetch_bytes=377487360 reduce_bytes=452984832 weight_movement_bytes=830472192 egress_bytes=2916432784 ingress_bytes=3743236096 link_bytes=3743236096 total_nvlink_bytes=6659668880 weight_to_token_ratio=0.142468 prefetch_share_of_ingress=0.100845 dedup_ratio=0.217266 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=18 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=17653 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.780106351e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49626 payload_rows_remote=41450 dup_rows=15910 dup_rows_remote=9316 dedup_ratio=0.242767 router_tokens_per_expert=[0, 0, 0, 0, 4067, 502, 9, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 0, 0, 4082, 0, 0, 0, 0, 4096, 3955, 0, 0, 0, 0, 0, 1, 3297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0] payload_rows_per_dst=[1022, 0, 8192, 7660, 8192, 8192, 8176, 8192] dup_rows_per_dst=[0, 0, 28, 0, 1366, 7912, 6594, 10] combine_rows_per_src=[1022, 0, 8192, 7660, 8192, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=169779200 dispatch_meta_bytes=203064 combine_payload_bytes=169779200 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=169982264 ingress_bytes=194945024 link_bytes=194945024 token_bytes=339761464 weight_movement_bytes=25165824 total_nvlink_bytes=364927288 weight_to_token_ratio=0.074069 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.483306386e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49292 payload_rows_remote=42326 dup_rows=16244 dup_rows_remote=13230 dedup_ratio=0.247864 router_tokens_per_expert=[1, 0, 0, 0, 4096, 2455, 0, 0, 4096, 0, 0, 4073, 0, 783, 0, 0, 209, 0, 1468, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 1768, 0, 0, 0, 0, 0, 2485, 0, 485, 0, 0, 0, 1690, 0, 72, 0, 0, 611, 0, 0, 2527, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4912, 8192, 2940, 8192, 6114, 3784, 6966, 8192] dup_rows_per_dst=[0, 9712, 414, 2, 2392, 710, 3014, 0] combine_rows_per_src=[4912, 8192, 2940, 8192, 6114, 3784, 6966, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=173367296 dispatch_meta_bytes=222224 combine_payload_bytes=173367296 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=173589520 ingress_bytes=248864768 link_bytes=248864768 token_bytes=346956816 weight_movement_bytes=75497472 total_nvlink_bytes=422454288 weight_to_token_ratio=0.217599 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=12416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.893573337e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46028 payload_rows_remote=37876 dup_rows=19508 dup_rows_remote=14648 dedup_ratio=0.297668 router_tokens_per_expert=[0, 0, 5, 0, 0, 3, 0, 0, 3787, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 4024, 0, 0, 15, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3263, 0, 0, 1813, 0, 0, 0, 0, 867, 0, 0, 1, 2381, 317, 0, 4032, 0, 0, 0, 0, 382, 0, 0, 3648, 0] payload_rows_per_dst=[16, 8188, 8068, 72, 8192, 5312, 8152, 8028] dup_rows_per_dst=[0, 7574, 10, 8, 6526, 48, 4860, 482] combine_rows_per_src=[16, 8188, 8068, 72, 8192, 5312, 8152, 8028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=155140096 dispatch_meta_bytes=210096 combine_payload_bytes=155140096 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=155350192 ingress_bytes=255803392 link_bytes=255803392 token_bytes=310490288 weight_movement_bytes=100663296 total_nvlink_bytes=411153584 weight_to_token_ratio=0.324208 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.946368249e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5791142912 prefetch_bytes=352321536 reduce_bytes=528482304 weight_movement_bytes=880803840 egress_bytes=2897376256 ingress_bytes=3774570496 link_bytes=3774570496 total_nvlink_bytes=6671946752 weight_to_token_ratio=0.152095 prefetch_share_of_ingress=0.093341 dedup_ratio=0.223759 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=21 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=17907 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.795007512e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none + [2026-09-21 08:31:45] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 223.3 | throughput per GPU (TFLOP/s/GPU): 144.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.006657E+01 | loss scale: 1.0 | grad norm: 101.471 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46846 payload_rows_remote=40952 dup_rows=18690 dup_rows_remote=16162 dedup_ratio=0.285187 router_tokens_per_expert=[2770, 619, 107, 139, 141, 343, 567, 79, 131, 889, 30, 3, 87, 281, 156, 1116, 34, 536, 850, 1353, 67, 476, 512, 374, 1106, 168, 29, 1068, 467, 718, 183, 341, 194, 390, 698, 307, 537, 1052, 268, 765, 349, 823, 712, 26, 2019, 756, 90, 394, 412, 543, 835, 23, 1831, 542, 350, 194, 1210, 15, 500, 61, 10, 485, 603, 34] payload_rows_per_dst=[6854, 6196, 5888, 5754, 5894, 4954, 4576, 6730] dup_rows_per_dst=[2676, 2852, 2516, 2406, 2528, 1346, 1222, 3144] combine_rows_per_src=[6854, 6196, 5888, 5754, 5894, 4954, 4576, 6730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47266 payload_rows_remote=41284 dup_rows=18270 dup_rows_remote=15698 dedup_ratio=0.278778 router_tokens_per_expert=[2449, 725, 172, 155, 32, 510, 367, 70, 82, 530, 70, 7, 107, 318, 162, 1263, 53, 447, 1347, 1316, 111, 642, 659, 177, 1001, 148, 44, 963, 577, 766, 210, 568, 173, 458, 636, 236, 523, 621, 356, 607, 446, 857, 635, 31, 1877, 1232, 36, 431, 866, 573, 622, 93, 1949, 425, 287, 232, 813, 12, 606, 40, 20, 268, 677, 82] payload_rows_per_dst=[6488, 6256, 6462, 5982, 5296, 5556, 4892, 6334] dup_rows_per_dst=[2472, 2720, 3042, 2572, 1924, 1780, 1304, 2456] combine_rows_per_src=[6488, 6256, 6462, 5982, 5296, 5556, 4892, 6334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38178 dup_rows=21100 dup_rows_remote=18720 dedup_ratio=0.321960 router_tokens_per_expert=[1980, 763, 53, 207, 123, 476, 624, 38, 256, 480, 330, 1, 38, 244, 109, 1345, 94, 339, 1247, 1383, 146, 888, 736, 283, 1155, 89, 74, 914, 521, 559, 69, 282, 170, 671, 463, 145, 622, 1114, 216, 486, 519, 790, 581, 22, 1884, 1059, 49, 711, 513, 556, 1000, 156, 1693, 401, 316, 350, 687, 11, 944, 232, 26, 241, 238, 56] payload_rows_per_dst=[3792, 4524, 4032, 5336, 6766, 6400, 7328, 6258] dup_rows_per_dst=[776, 1082, 940, 1990, 3774, 3556, 6602, 2380] combine_rows_per_src=[3792, 4524, 4032, 5336, 6766, 6400, 7328, 6258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46692 payload_rows_remote=40536 dup_rows=18844 dup_rows_remote=16090 dedup_ratio=0.287537 router_tokens_per_expert=[2529, 747, 126, 147, 63, 484, 276, 44, 232, 569, 98, 2, 108, 345, 112, 1412, 78, 372, 1123, 1443, 44, 387, 643, 351, 932, 134, 105, 649, 706, 1219, 388, 410, 59, 519, 762, 103, 874, 905, 210, 825, 324, 1192, 792, 39, 1641, 532, 54, 475, 404, 900, 937, 82, 1577, 254, 320, 169, 667, 7, 670, 148, 31, 355, 531, 132] payload_rows_per_dst=[6566, 6156, 6122, 5992, 5830, 5154, 4752, 6120] dup_rows_per_dst=[2266, 2754, 2760, 3094, 2684, 1662, 1380, 2244] combine_rows_per_src=[6566, 6156, 6122, 5992, 5830, 5154, 4752, 6120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46078 payload_rows_remote=39590 dup_rows=19458 dup_rows_remote=15898 dedup_ratio=0.296906 router_tokens_per_expert=[1990, 864, 129, 125, 137, 562, 484, 190, 318, 702, 38, 1, 69, 514, 419, 1264, 174, 754, 1483, 1197, 130, 373, 753, 160, 1034, 195, 142, 688, 528, 822, 176, 388, 244, 485, 966, 157, 484, 647, 289, 471, 389, 834, 1021, 8, 1818, 904, 51, 429, 470, 722, 775, 124, 1572, 476, 318, 134, 760, 5, 518, 186, 34, 360, 262, 52] payload_rows_per_dst=[6236, 6366, 6488, 5480, 5380, 5404, 4722, 6002] dup_rows_per_dst=[2726, 3428, 3560, 2466, 2106, 1868, 1316, 1988] combine_rows_per_src=[6236, 6366, 6488, 5480, 5380, 5404, 4722, 6002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45846 payload_rows_remote=40204 dup_rows=19690 dup_rows_remote=17354 dedup_ratio=0.300446 router_tokens_per_expert=[2001, 619, 346, 250, 179, 450, 768, 130, 105, 561, 200, 2, 40, 277, 225, 1564, 305, 333, 629, 1683, 98, 269, 676, 190, 743, 110, 231, 1385, 409, 713, 101, 451, 398, 313, 382, 95, 859, 567, 192, 317, 1052, 959, 839, 24, 1445, 612, 91, 412, 567, 925, 1114, 234, 1757, 384, 260, 272, 819, 4, 701, 274, 19, 351, 414, 73] payload_rows_per_dst=[6426, 5900, 5668, 5782, 4718, 5642, 5924, 5786] dup_rows_per_dst=[3060, 2488, 2620, 2504, 1606, 2336, 2662, 2414] combine_rows_per_src=[6426, 5900, 5668, 5782, 4718, 5642, 5924, 5786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45494 payload_rows_remote=38970 dup_rows=20042 dup_rows_remote=17098 dedup_ratio=0.305817 router_tokens_per_expert=[2178, 813, 109, 108, 252, 767, 382, 125, 345, 791, 47, 1, 79, 299, 152, 687, 79, 639, 941, 1414, 176, 687, 622, 420, 816, 144, 46, 1098, 545, 589, 147, 256, 291, 411, 478, 108, 419, 598, 222, 570, 284, 970, 886, 8, 1870, 863, 108, 701, 572, 526, 1073, 119, 1940, 559, 225, 436, 913, 15, 710, 191, 13, 288, 487, 160] payload_rows_per_dst=[6524, 6230, 6412, 5328, 4724, 6520, 5220, 4536] dup_rows_per_dst=[2944, 2452, 3544, 1954, 1470, 4680, 1800, 1198] combine_rows_per_src=[6524, 6230, 6412, 5328, 4724, 6520, 5220, 4536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=167739392 dispatch_meta_bytes=228456 combine_payload_bytes=167739392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=167967848 ingress_bytes=192905216 link_bytes=192905216 token_bytes=335707240 weight_movement_bytes=25165824 total_nvlink_bytes=360873064 weight_to_token_ratio=0.074964 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.467785804e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=169099264 dispatch_meta_bytes=227928 combine_payload_bytes=169099264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=169327192 ingress_bytes=194265088 link_bytes=194265088 token_bytes=338426456 weight_movement_bytes=25165824 total_nvlink_bytes=363592280 weight_to_token_ratio=0.074361 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.478132859e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156377088 dispatch_meta_bytes=227592 combine_payload_bytes=156377088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=156604680 ingress_bytes=181542912 link_bytes=181542912 token_bytes=312981768 weight_movement_bytes=25165824 total_nvlink_bytes=338147592 weight_to_token_ratio=0.080407 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.381331799e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166035456 dispatch_meta_bytes=226504 combine_payload_bytes=166035456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166261960 ingress_bytes=191201280 link_bytes=191201280 token_bytes=332297416 weight_movement_bytes=25165824 total_nvlink_bytes=357463240 weight_to_token_ratio=0.075733 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.454820820e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162160640 dispatch_meta_bytes=221952 combine_payload_bytes=162160640 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=162382592 ingress_bytes=237658112 link_bytes=237658112 token_bytes=324543232 weight_movement_bytes=75497472 total_nvlink_bytes=400040704 weight_to_token_ratio=0.232627 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.808303634e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164675584 dispatch_meta_bytes=230232 combine_payload_bytes=164675584 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=164905816 ingress_bytes=215007232 link_bytes=215007232 token_bytes=329581400 weight_movement_bytes=50331648 total_nvlink_bytes=379913048 weight_to_token_ratio=0.152714 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.635956608e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44844 payload_rows_remote=38112 dup_rows=20692 dup_rows_remote=15588 dedup_ratio=0.315735 router_tokens_per_expert=[2198, 795, 73, 278, 76, 556, 528, 22, 305, 957, 166, 0, 33, 360, 147, 1098, 40, 150, 1074, 1581, 98, 715, 979, 288, 803, 126, 184, 776, 679, 872, 205, 465, 176, 710, 550, 240, 529, 768, 244, 620, 480, 686, 628, 3, 1643, 1066, 14, 489, 646, 599, 852, 138, 1499, 402, 437, 392, 1064, 15, 559, 184, 14, 154, 257, 83] payload_rows_per_dst=[4446, 4652, 4238, 5852, 6874, 6104, 6732, 5946] dup_rows_per_dst=[1284, 1480, 1244, 3044, 3962, 2574, 5104, 2000] combine_rows_per_src=[4446, 4652, 4238, 5852, 6874, 6104, 6732, 5946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159621120 dispatch_meta_bytes=224272 combine_payload_bytes=159621120 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159845392 ingress_bytes=235118592 link_bytes=235118592 token_bytes=319466512 weight_movement_bytes=75497472 total_nvlink_bytes=394963984 weight_to_token_ratio=0.236324 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788980821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=156106752 dispatch_meta_bytes=214800 combine_payload_bytes=156106752 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=156321552 ingress_bytes=206438400 link_bytes=206438400 token_bytes=312428304 weight_movement_bytes=50331648 total_nvlink_bytes=362759952 weight_to_token_ratio=0.161098 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7296 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.570757698e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49622 payload_rows_remote=42096 dup_rows=15914 dup_rows_remote=13972 dedup_ratio=0.242828 router_tokens_per_expert=[4, 497, 3617, 0, 0, 0, 195, 0, 13, 0, 52, 1026, 1517, 7, 38, 0, 1266, 0, 0, 2, 19, 3759, 0, 668, 0, 49, 0, 52, 998, 2, 4, 7, 471, 0, 171, 0, 0, 4, 1, 470, 51, 3999, 2311, 0, 3352, 0, 58, 59, 3152, 1666, 0, 767, 5, 0, 0, 1044, 435, 673, 19, 0, 264, 0, 4, 0] payload_rows_per_dst=[1382, 5240, 7774, 8030, 7526, 7736, 7502, 4432] dup_rows_per_dst=[10, 1402, 2318, 2192, 1942, 3926, 3678, 446] combine_rows_per_src=[1382, 5240, 7774, 8030, 7526, 7736, 7502, 4432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45256 payload_rows_remote=37122 dup_rows=20280 dup_rows_remote=14384 dedup_ratio=0.309448 router_tokens_per_expert=[5, 1083, 2273, 0, 0, 0, 185, 17, 200, 7, 1701, 527, 189, 0, 18, 59, 259, 0, 0, 0, 4, 2301, 0, 4090, 1, 0, 1, 165, 657, 4, 42, 28, 3756, 0, 86, 138, 7, 0, 13, 222, 110, 619, 346, 0, 1422, 0, 1240, 37, 2661, 2427, 0, 1500, 13, 1, 0, 3903, 9, 6, 24, 0, 412, 0, 0, 0] payload_rows_per_dst=[2450, 4686, 8188, 2782, 8030, 5060, 5926, 8134] dup_rows_per_dst=[130, 716, 5120, 252, 4960, 1250, 1956, 5896] combine_rows_per_src=[2450, 4686, 8188, 2782, 8030, 5060, 5926, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48978 payload_rows_remote=42092 dup_rows=16558 dup_rows_remote=15206 dedup_ratio=0.252655 router_tokens_per_expert=[0, 2617, 3968, 0, 2, 40, 1323, 0, 67, 0, 231, 604, 69, 141, 0, 0, 132, 0, 153, 0, 62, 3419, 2, 3007, 0, 9, 2, 11, 1224, 1, 74, 47, 2061, 0, 640, 0, 32, 0, 0, 16, 128, 1854, 2964, 0, 61, 14, 101, 25, 1935, 1951, 21, 422, 0, 8, 25, 3137, 21, 1, 3, 2, 0, 0, 141, 0] payload_rows_per_dst=[8142, 6886, 7204, 6504, 4984, 5214, 3630, 6414] dup_rows_per_dst=[9402, 1352, 332, 2942, 514, 606, 1214, 196] combine_rows_per_src=[8142, 6886, 7204, 6504, 4984, 5214, 3630, 6414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50392 payload_rows_remote=44582 dup_rows=15144 dup_rows_remote=13116 dedup_ratio=0.231079 router_tokens_per_expert=[49, 2813, 2861, 0, 0, 0, 721, 0, 44, 0, 24, 1539, 59, 0, 1, 1, 60, 1, 1, 0, 0, 2625, 37, 3636, 5, 192, 0, 5, 384, 19, 9, 69, 1643, 12, 63, 9, 0, 1, 39, 639, 2, 2532, 1049, 0, 93, 1, 2315, 459, 2370, 772, 0, 3315, 0, 0, 0, 1940, 35, 255, 3, 34, 24, 0, 8, 0] payload_rows_per_dst=[6260, 7734, 5362, 5658, 7334, 5810, 7592, 4642] dup_rows_per_dst=[906, 2874, 86, 772, 3200, 2028, 4788, 490] combine_rows_per_src=[6260, 7734, 5362, 5658, 7334, 5810, 7592, 4642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48488 payload_rows_remote=40376 dup_rows=17048 dup_rows_remote=13390 dedup_ratio=0.260132 router_tokens_per_expert=[132, 1037, 196, 1, 0, 1, 488, 3, 704, 0, 0, 1001, 458, 0, 63, 32, 210, 20, 0, 23, 72, 1924, 1, 3723, 41, 31, 0, 1585, 38, 4, 39, 130, 2154, 1, 4, 1, 0, 1, 16, 1739, 32, 4017, 2203, 0, 3654, 0, 13, 329, 2167, 1326, 0, 177, 0, 0, 23, 2348, 120, 416, 10, 1, 8, 0, 51, 0] payload_rows_per_dst=[3230, 7962, 4328, 8112, 6214, 7868, 5406, 5368] dup_rows_per_dst=[94, 4000, 172, 3658, 2010, 4594, 1980, 540] combine_rows_per_src=[3230, 7962, 4328, 8112, 6214, 7868, 5406, 5368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50516 payload_rows_remote=46688 dup_rows=15020 dup_rows_remote=14942 dedup_ratio=0.229187 router_tokens_per_expert=[0, 2694, 351, 3, 9, 0, 0, 0, 332, 4, 34, 851, 43, 0, 0, 2, 73, 0, 186, 0, 0, 1694, 0, 4059, 0, 0, 2, 10, 717, 22, 252, 696, 2333, 0, 72, 0, 50, 0, 96, 2125, 5, 851, 1294, 2, 3722, 13, 1001, 0, 3239, 728, 3, 727, 0, 0, 0, 2841, 209, 23, 3, 0, 1369, 0, 28, 0] payload_rows_per_dst=[5652, 8134, 3828, 4142, 6812, 7820, 7376, 6752] dup_rows_per_dst=[448, 2516, 78, 958, 2554, 4254, 2018, 2194] combine_rows_per_src=[5652, 8134, 3828, 4142, 6812, 7820, 7376, 6752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=172425216 dispatch_meta_bytes=224272 combine_payload_bytes=172425216 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=172649488 ingress_bytes=197591040 link_bytes=197591040 token_bytes=345074704 weight_movement_bytes=25165824 total_nvlink_bytes=370240528 weight_to_token_ratio=0.072929 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.503439511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=152051712 dispatch_meta_bytes=206024 combine_payload_bytes=152051712 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=152257736 ingress_bytes=202383360 link_bytes=202383360 token_bytes=304309448 weight_movement_bytes=50331648 total_nvlink_bytes=354641096 weight_to_token_ratio=0.165396 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.539903529e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=172408832 dispatch_meta_bytes=229192 combine_payload_bytes=172408832 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=172638024 ingress_bytes=197574656 link_bytes=197574656 token_bytes=345046856 weight_movement_bytes=25165824 total_nvlink_bytes=370212680 weight_to_token_ratio=0.072935 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.503314847e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=40690 payload_rows_remote=32500 dup_rows=24846 dup_rows_remote=10002 dedup_ratio=0.379120 router_tokens_per_expert=[0, 3883, 3779, 0, 14, 0, 2355, 0, 109, 18, 209, 720, 11, 0, 0, 0, 20, 0, 14, 1, 0, 2317, 0, 4050, 0, 41, 0, 129, 296, 28, 94, 6, 570, 16, 381, 0, 0, 0, 79, 547, 0, 1647, 2355, 7, 117, 0, 371, 86, 1516, 1486, 0, 10, 1, 0, 0, 3642, 150, 6, 16, 0, 1112, 0, 559, 0] payload_rows_per_dst=[8190, 8186, 5638, 6738, 2768, 1550, 22, 7598] dup_rows_per_dst=[14844, 2048, 1594, 2454, 418, 116, 0, 3372] combine_rows_per_src=[8190, 8186, 5638, 6738, 2768, 1550, 22, 7598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=182607872 dispatch_meta_bytes=230792 combine_payload_bytes=182607872 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=182838664 ingress_bytes=258105344 link_bytes=258105344 token_bytes=365446536 weight_movement_bytes=75497472 total_nvlink_bytes=440944008 weight_to_token_ratio=0.206590 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.963883444e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=165380096 dispatch_meta_bytes=215064 combine_payload_bytes=165380096 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=165595160 ingress_bytes=215711744 link_bytes=215711744 token_bytes=330975256 weight_movement_bytes=50331648 total_nvlink_bytes=381306904 weight_to_token_ratio=0.152071 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=19456 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.641317131e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=191234048 dispatch_meta_bytes=246520 combine_payload_bytes=191234048 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=191480568 ingress_bytes=241565696 link_bytes=241565696 token_bytes=382714616 weight_movement_bytes=50331648 total_nvlink_bytes=433046264 weight_to_token_ratio=0.131512 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.838035833e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48956 payload_rows_remote=40978 dup_rows=16580 dup_rows_remote=12110 dedup_ratio=0.252991 router_tokens_per_expert=[0, 2035, 2721, 29, 0, 1, 133, 0, 28, 0, 115, 29, 110, 16, 0, 13, 4, 24, 12, 0, 1, 3937, 1, 3367, 0, 132, 1, 0, 1046, 0, 4, 150, 142, 0, 30, 0, 9, 0, 43, 190, 178, 3838, 2565, 0, 1092, 2, 11, 4, 1757, 3787, 0, 2437, 0, 0, 0, 2459, 38, 180, 3, 8, 3, 0, 83, 0] payload_rows_per_dst=[4330, 1572, 8122, 8118, 5702, 6360, 7978, 6774] dup_rows_per_dst=[66, 228, 5392, 2224, 568, 1344, 4470, 2288] combine_rows_per_src=[4330, 1572, 8122, 8118, 5702, 6360, 7978, 6774] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=133120000 dispatch_meta_bytes=170008 combine_payload_bytes=133120000 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=133290008 ingress_bytes=183451648 link_bytes=183451648 token_bytes=266410008 weight_movement_bytes=50331648 total_nvlink_bytes=316741656 weight_to_token_ratio=0.188926 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.395855075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53698 payload_rows_remote=45710 dup_rows=11838 dup_rows_remote=11730 dedup_ratio=0.180634 router_tokens_per_expert=[484, 0, 0, 0, 8, 1, 0, 0, 0, 433, 6, 461, 0, 0, 0, 0, 1, 3744, 9, 3854, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 47, 0, 0, 41, 13, 3990, 0, 1, 1, 2, 0, 0, 0, 0, 3877, 448, 4016, 677, 3, 1, 56, 0, 0, 3, 47, 0, 3454, 264, 0, 0, 0, 0, 0, 2730, 0] payload_rows_per_dst=[986, 8160, 8090, 8190, 7988, 8188, 6080, 6016] dup_rows_per_dst=[0, 1394, 7126, 96, 108, 2100, 84, 930] combine_rows_per_src=[986, 8160, 8090, 8190, 7988, 8188, 6080, 6016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50780 payload_rows_remote=42588 dup_rows=14756 dup_rows_remote=10708 dedup_ratio=0.225159 router_tokens_per_expert=[1219, 0, 49, 0, 59, 2, 0, 0, 180, 613, 1384, 95, 0, 0, 0, 40, 2, 4052, 3, 4034, 0, 0, 0, 1, 0, 0, 0, 0, 924, 493, 0, 0, 37, 3, 3968, 0, 2127, 0, 2248, 0, 7, 0, 0, 3808, 10, 2654, 0, 0, 1, 297, 223, 0, 0, 0, 89, 1592, 23, 6, 0, 1, 0, 4, 2429, 91] payload_rows_per_dst=[8116, 8192, 8092, 5094, 6992, 5336, 3898, 5060] dup_rows_per_dst=[2646, 4048, 3584, 1928, 1990, 6, 506, 48] combine_rows_per_src=[8116, 8192, 8092, 5094, 6992, 5336, 3898, 5060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51026 payload_rows_remote=42894 dup_rows=14510 dup_rows_remote=11610 dedup_ratio=0.221405 router_tokens_per_expert=[3, 47, 0, 22, 152, 0, 1, 1, 5, 0, 0, 3274, 0, 0, 30, 0, 0, 4093, 0, 511, 55, 0, 0, 0, 0, 1, 0, 0, 505, 2456, 0, 0, 0, 2, 4050, 0, 2484, 0, 1, 0, 17, 0, 0, 3115, 2507, 6, 0, 1, 0, 3775, 0, 0, 1, 137, 0, 3998, 22, 0, 0, 0, 1, 0, 1425, 70] payload_rows_per_dst=[406, 6558, 8190, 5272, 6486, 8130, 7852, 8132] dup_rows_per_dst=[46, 60, 1128, 652, 1178, 7828, 718, 2900] combine_rows_per_src=[406, 6558, 8190, 5272, 6486, 8130, 7852, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45854 payload_rows_remote=38066 dup_rows=19682 dup_rows_remote=19046 dedup_ratio=0.300323 router_tokens_per_expert=[1644, 1, 2, 216, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 1, 342, 0, 0, 0, 0, 0, 9, 0, 3, 3975, 23, 0, 84, 2503, 84, 4018, 0, 2227, 1, 487, 1, 0, 0, 0, 4048, 77, 454, 194, 0, 0, 298, 7, 2, 43, 0, 0, 4005, 2231, 1, 0, 0, 0, 0, 1671, 6] payload_rows_per_dst=[3722, 1146, 8192, 8104, 8148, 7788, 690, 8064] dup_rows_per_dst=[8, 0, 686, 84, 10494, 636, 10, 7764] combine_rows_per_src=[3722, 1146, 8192, 8104, 8148, 7788, 690, 8064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51336 payload_rows_remote=43180 dup_rows=14200 dup_rows_remote=14016 dedup_ratio=0.216675 router_tokens_per_expert=[3106, 0, 289, 0, 8, 0, 7, 0, 2, 10, 0, 3780, 25, 0, 8, 0, 0, 4085, 0, 2405, 0, 0, 1, 0, 2, 0, 0, 0, 4076, 92, 0, 0, 1, 0, 184, 0, 2095, 0, 21, 0, 2, 0, 0, 1546, 943, 121, 16, 0, 0, 2926, 0, 148, 2, 1, 0, 3210, 491, 0, 0, 0, 0, 0, 3165, 0] payload_rows_per_dst=[8032, 7978, 5588, 8156, 4560, 2116, 8056, 6850] dup_rows_per_dst=[6014, 2764, 168, 184, 42, 48, 4518, 462] combine_rows_per_src=[8032, 7978, 5588, 8156, 4560, 2116, 8056, 6850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=167845888 dispatch_meta_bytes=212352 combine_payload_bytes=167845888 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=168058240 ingress_bytes=293675008 link_bytes=293675008 token_bytes=335904128 weight_movement_bytes=125829120 total_nvlink_bytes=461733248 weight_to_token_ratio=0.374598 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=11904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.234527488e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47254 payload_rows_remote=46642 dup_rows=18282 dup_rows_remote=18260 dedup_ratio=0.278961 router_tokens_per_expert=[3805, 1, 7, 0, 17, 0, 1, 2, 3, 217, 0, 90, 11, 2, 0, 0, 11, 3891, 277, 0, 0, 26, 3, 0, 0, 0, 1, 47, 2937, 2986, 0, 1, 79, 0, 1350, 0, 2620, 0, 1397, 0, 0, 0, 4, 4066, 1747, 1608, 75, 0, 6, 289, 2, 0, 1, 0, 316, 4017, 80, 0, 0, 2, 0, 0, 773, 0] payload_rows_per_dst=[7968, 8186, 612, 7590, 7320, 5712, 8166, 1700] dup_rows_per_dst=[7480, 592, 22, 4354, 3572, 1156, 1096, 10] combine_rows_per_src=[7968, 8186, 612, 7590, 7320, 5712, 8166, 1700] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=187228160 dispatch_meta_bytes=229760 combine_payload_bytes=187228160 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=187457920 ingress_bytes=287891456 link_bytes=287891456 token_bytes=374686080 weight_movement_bytes=100663296 total_nvlink_bytes=475349376 weight_to_token_ratio=0.268660 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.190521340e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=174440448 dispatch_meta_bytes=213184 combine_payload_bytes=174440448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=174653632 ingress_bytes=199606272 link_bytes=199606272 token_bytes=349094080 weight_movement_bytes=25165824 total_nvlink_bytes=374259904 weight_to_token_ratio=0.072089 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.518773098e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=175693824 dispatch_meta_bytes=218016 combine_payload_bytes=175693824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=175911840 ingress_bytes=200859648 link_bytes=200859648 token_bytes=351605664 weight_movement_bytes=25165824 total_nvlink_bytes=376771488 weight_to_token_ratio=0.071574 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.528309841e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=41442 payload_rows_remote=33264 dup_rows=24094 dup_rows_remote=17534 dedup_ratio=0.367645 router_tokens_per_expert=[3269, 0, 0, 0, 1, 0, 0, 11, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 4088, 14, 2178, 0, 0, 0, 2, 0, 0, 0, 0, 4090, 0, 0, 0, 13, 2, 696, 5, 460, 0, 120, 0, 0, 0, 0, 3990, 1, 1, 0, 0, 1032, 3506, 3, 5, 4, 0, 0, 4062, 415, 0, 665, 0, 0, 0, 46, 0] payload_rows_per_dst=[8178, 8188, 5580, 8180, 1182, 4, 8192, 1938] dup_rows_per_dst=[6560, 7970, 200, 0, 18, 0, 9032, 314] combine_rows_per_src=[8178, 8188, 5580, 8180, 1182, 4, 8192, 1938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=176865280 dispatch_meta_bytes=228784 combine_payload_bytes=176865280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=177094064 ingress_bytes=202031104 link_bytes=202031104 token_bytes=353959344 weight_movement_bytes=25165824 total_nvlink_bytes=379125168 weight_to_token_ratio=0.071098 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.537223267e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=155918336 dispatch_meta_bytes=228448 combine_payload_bytes=155918336 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=156146784 ingress_bytes=206249984 link_bytes=206249984 token_bytes=312065120 weight_movement_bytes=50331648 total_nvlink_bytes=362396768 weight_to_token_ratio=0.161286 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.569324070e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=191045632 dispatch_meta_bytes=259608 combine_payload_bytes=191045632 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=191305240 ingress_bytes=241377280 link_bytes=241377280 token_bytes=382350872 weight_movement_bytes=50331648 total_nvlink_bytes=432682520 weight_to_token_ratio=0.131637 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.836602205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=136249344 dispatch_meta_bytes=203192 combine_payload_bytes=136249344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=136452536 ingress_bytes=161415168 link_bytes=161415168 token_bytes=272701880 weight_movement_bytes=25165824 total_nvlink_bytes=297867704 weight_to_token_ratio=0.092283 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.228182924e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49362 payload_rows_remote=42968 dup_rows=16174 dup_rows_remote=15930 dedup_ratio=0.246796 router_tokens_per_expert=[2298, 0, 80, 0, 309, 0, 0, 0, 0, 146, 0, 1, 3, 0, 0, 0, 0, 4074, 0, 62, 1, 2, 0, 0, 0, 0, 0, 0, 3825, 1, 0, 0, 1, 0, 2167, 0, 515, 1, 45, 0, 0, 4, 0, 3628, 3589, 1023, 0, 0, 1, 3171, 4, 9, 133, 1, 0, 4065, 3560, 23, 0, 0, 0, 0, 26, 0] payload_rows_per_dst=[5206, 300, 8190, 7650, 5252, 8192, 6394, 8178] dup_rows_per_dst=[168, 0, 88, 2, 206, 8296, 244, 7170] combine_rows_per_src=[5206, 300, 8190, 7650, 5252, 8192, 6394, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60416 payload_rows_remote=53716 dup_rows=5120 dup_rows_remote=5094 dedup_ratio=0.078125 router_tokens_per_expert=[0, 0, 97, 0, 4096, 4053, 594, 3909, 0, 0, 237, 0, 0, 0, 0, 1434, 3422, 0, 80, 0, 0, 9, 0, 0, 0, 0, 0, 3176, 0, 4080, 0, 0, 20, 4096, 6, 3329, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 11, 0, 32, 0, 72] payload_rows_per_dst=[8122, 8192, 6844, 6352, 6700, 7828, 8192, 8186] dup_rows_per_dst=[1366, 3342, 178, 0, 26, 0, 4, 204] combine_rows_per_src=[8122, 8192, 6844, 6352, 6700, 7828, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56276 payload_rows_remote=48268 dup_rows=9260 dup_rows_remote=9168 dedup_ratio=0.141296 router_tokens_per_expert=[0, 0, 1216, 0, 4074, 3652, 476, 4055, 0, 60, 0, 4, 22, 0, 0, 0, 2972, 20, 3979, 87, 250, 0, 0, 1, 0, 0, 223, 141, 0, 3458, 4, 0, 123, 3964, 0, 3771, 0, 131, 0, 0, 2, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 18, 0, 1, 0, 1] payload_rows_per_dst=[8172, 8008, 8192, 726, 7892, 8192, 8154, 6940] dup_rows_per_dst=[2516, 92, 6426, 10, 158, 34, 0, 24] combine_rows_per_src=[8172, 8008, 8192, 726, 7892, 8192, 8154, 6940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50414 payload_rows_remote=42226 dup_rows=15122 dup_rows_remote=9746 dedup_ratio=0.230743 router_tokens_per_expert=[448, 3, 421, 0, 4096, 4061, 1584, 3765, 0, 0, 90, 0, 0, 0, 0, 0, 2646, 0, 2949, 0, 1563, 0, 0, 0, 230, 0, 3, 1342, 0, 3764, 0, 15, 332, 2166, 9, 191, 0, 17, 0, 30, 4, 0, 1, 1, 0, 4, 0, 3, 0, 0, 0, 12, 0, 0, 0, 0, 0, 112, 39, 0, 2866, 0, 0, 1] payload_rows_per_dst=[3246, 180, 7882, 8106, 7068, 7552, 8192, 8188] dup_rows_per_dst=[1666, 0, 536, 2500, 4320, 4, 720, 5376] combine_rows_per_src=[3246, 180, 7882, 8106, 7068, 7552, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=175996928 dispatch_meta_bytes=235592 combine_payload_bytes=175996928 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=176232520 ingress_bytes=226328576 link_bytes=226328576 token_bytes=352229448 weight_movement_bytes=50331648 total_nvlink_bytes=402561096 weight_to_token_ratio=0.142894 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.722098955e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56920 payload_rows_remote=48730 dup_rows=8616 dup_rows_remote=5980 dedup_ratio=0.131470 router_tokens_per_expert=[0, 1, 1, 0, 4072, 3923, 45, 4082, 0, 0, 0, 0, 0, 421, 0, 58, 3306, 0, 3115, 0, 0, 5, 0, 43, 1436, 0, 189, 3788, 0, 4055, 0, 0, 19, 4047, 1, 67, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 6, 1, 5, 0, 16, 25, 0, 0, 0, 0, 0, 7, 0, 0, 30, 1, 0, 0] payload_rows_per_dst=[7936, 8182, 7934, 8190, 172, 8174, 8178, 8154] dup_rows_per_dst=[4, 870, 5004, 2636, 2, 10, 58, 32] combine_rows_per_src=[7936, 8182, 7934, 8190, 172, 8174, 8178, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57792 payload_rows_remote=49600 dup_rows=7744 dup_rows_remote=7736 dedup_ratio=0.118164 router_tokens_per_expert=[1, 0, 586, 1, 4044, 4067, 368, 4096, 0, 2, 0, 0, 0, 0, 0, 1582, 881, 0, 3304, 0, 0, 2, 32, 0, 153, 0, 0, 3928, 0, 4093, 0, 2, 3, 4092, 0, 1247, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 5, 0, 0, 93, 1, 0, 0, 172] payload_rows_per_dst=[8192, 8190, 4816, 7892, 4212, 8192, 8106, 8192] dup_rows_per_dst=[1854, 3162, 328, 274, 1582, 8, 0, 536] combine_rows_per_src=[8192, 8190, 4816, 7892, 4212, 8192, 8106, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=220020736 dispatch_meta_bytes=235240 combine_payload_bytes=220020736 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=220255976 ingress_bytes=270352384 link_bytes=270352384 token_bytes=440276712 weight_movement_bytes=50331648 total_nvlink_bytes=490608360 weight_to_token_ratio=0.114318 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19328 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.057069267e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61886 payload_rows_remote=53704 dup_rows=3650 dup_rows_remote=1416 dedup_ratio=0.055695 router_tokens_per_expert=[26, 1, 0, 0, 3786, 2979, 392, 3999, 0, 91, 0, 0, 0, 0, 0, 242, 1223, 0, 3923, 0, 62, 0, 0, 0, 0, 1, 41, 4028, 0, 3340, 1, 7, 0, 3769, 875, 3459, 0, 0, 0, 0, 13, 0, 0, 64, 0, 10, 26, 0, 1, 19, 0, 0, 0, 121, 0, 0, 0, 0, 2, 54, 47, 0, 0, 166] payload_rows_per_dst=[6748, 7836, 8182, 8064, 8008, 8130, 7846, 7072] dup_rows_per_dst=[48, 368, 2234, 92, 660, 94, 8, 146] combine_rows_per_src=[6748, 7836, 8182, 8064, 8008, 8130, 7846, 7072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=197705728 dispatch_meta_bytes=229744 combine_payload_bytes=197705728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197935472 ingress_bytes=222871552 link_bytes=222871552 token_bytes=395641200 weight_movement_bytes=25165824 total_nvlink_bytes=420807024 weight_to_token_ratio=0.063608 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=26496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.695794997e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=172957696 dispatch_meta_bytes=207888 combine_payload_bytes=172957696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=173165584 ingress_bytes=198123520 link_bytes=198123520 token_bytes=346123280 weight_movement_bytes=25165824 total_nvlink_bytes=371289104 weight_to_token_ratio=0.072708 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=29056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.507491068e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56350 payload_rows_remote=48158 dup_rows=9186 dup_rows_remote=5268 dedup_ratio=0.140167 router_tokens_per_expert=[6, 0, 553, 0, 4096, 4088, 957, 3916, 1, 0, 73, 0, 0, 1239, 1, 511, 1207, 0, 0, 0, 99, 0, 0, 0, 71, 0, 0, 4093, 0, 4096, 1, 1, 136, 4094, 21, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 21] payload_rows_per_dst=[8192, 8190, 2238, 8192, 7230, 7316, 8192, 6800] dup_rows_per_dst=[3918, 3648, 0, 1574, 34, 2, 10, 0] combine_rows_per_src=[8192, 8190, 2238, 8192, 7230, 7316, 8192, 6800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=199598080 dispatch_meta_bytes=218840 combine_payload_bytes=199598080 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=199816920 ingress_bytes=249929728 link_bytes=249929728 token_bytes=399415000 weight_movement_bytes=50331648 total_nvlink_bytes=449746648 weight_to_token_ratio=0.126013 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=23424 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.901676452e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=203161600 dispatch_meta_bytes=229344 combine_payload_bytes=203161600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203390944 ingress_bytes=228327424 link_bytes=228327424 token_bytes=406552544 weight_movement_bytes=25165824 total_nvlink_bytes=431718368 weight_to_token_ratio=0.061901 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.737307879e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=219971584 dispatch_meta_bytes=220480 combine_payload_bytes=219971584 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=220192064 ingress_bytes=270303232 link_bytes=270303232 token_bytes=440163648 weight_movement_bytes=50331648 total_nvlink_bytes=490495296 weight_to_token_ratio=0.114348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.056695277e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53462 payload_rows_remote=45290 dup_rows=12074 dup_rows_remote=5388 dedup_ratio=0.184235 router_tokens_per_expert=[0, 0, 572, 0, 98, 0, 120, 0, 0, 0, 8, 0, 0, 0, 0, 4028, 0, 1, 4096, 4050, 0, 0, 1795, 0, 0, 0, 184, 4094, 0, 0, 0, 1057, 0, 1, 0, 0, 195, 3156, 4076, 1, 0, 0, 0, 99, 7, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 905, 0, 0, 130, 0, 2] payload_rows_per_dst=[8188, 8072, 8188, 2476, 8172, 8186, 8192, 1988] dup_rows_per_dst=[1580, 0, 3504, 6, 6686, 210, 2, 86] combine_rows_per_src=[8188, 8072, 8188, 2476, 8172, 8186, 8192, 1988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=197255168 dispatch_meta_bytes=213704 combine_payload_bytes=197255168 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=197468872 ingress_bytes=323084288 link_bytes=323084288 token_bytes=394724040 weight_movement_bytes=125829120 total_nvlink_bytes=520553160 weight_to_token_ratio=0.318777 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.458298128e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49202 payload_rows_remote=41330 dup_rows=16334 dup_rows_remote=16330 dedup_ratio=0.249237 router_tokens_per_expert=[0, 0, 277, 0, 3926, 4039, 0, 4086, 37, 0, 0, 211, 19, 0, 0, 0, 2317, 5, 3355, 0, 44, 5, 0, 0, 0, 6, 0, 2919, 0, 2824, 0, 1, 0, 4065, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 914, 0, 0, 0, 0, 1, 8, 0, 1, 0, 0, 0, 2, 0, 2, 0, 0, 7, 10, 0, 0] payload_rows_per_dst=[3114, 4804, 4734, 6620, 8186, 8186, 7872, 5686] dup_rows_per_dst=[442, 472, 8, 4308, 9286, 1814, 4, 0] combine_rows_per_src=[3114, 4804, 4734, 6620, 8186, 8186, 7872, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46030 payload_rows_remote=37838 dup_rows=19506 dup_rows_remote=11342 dedup_ratio=0.297638 router_tokens_per_expert=[0, 0, 276, 2530, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 4094, 0, 0, 4083, 4077, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 7, 0, 3796, 1, 3489, 25, 279, 0, 26, 0, 0, 0, 2, 1805, 2, 85, 27, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 4, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 222, 1042, 7976, 4178, 8114, 8120] dup_rows_per_dst=[5612, 8164, 0, 2, 5624, 46, 58, 0] combine_rows_per_src=[8186, 8192, 222, 1042, 7976, 4178, 8114, 8120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46908 payload_rows_remote=40048 dup_rows=18628 dup_rows_remote=16448 dedup_ratio=0.284241 router_tokens_per_expert=[2038, 0, 0, 0, 9, 0, 0, 121, 127, 0, 236, 0, 12, 0, 0, 4, 2, 0, 4068, 3978, 4, 0, 0, 411, 8, 35, 3591, 4074, 0, 5, 23, 0, 0, 0, 0, 0, 1547, 10, 1088, 0, 398, 0, 0, 0, 0, 11, 0, 3513, 3873, 2, 148, 0, 0, 0, 0, 0, 43, 1, 3361, 27, 0, 0, 0, 0] payload_rows_per_dst=[4076, 670, 8190, 8190, 3112, 7650, 8160, 6860] dup_rows_per_dst=[260, 88, 600, 7282, 2, 194, 8022, 2180] combine_rows_per_src=[4076, 670, 8190, 8190, 3112, 7650, 8160, 6860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=185507840 dispatch_meta_bytes=202712 combine_payload_bytes=185507840 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=185710552 ingress_bytes=286171136 link_bytes=286171136 token_bytes=371218392 weight_movement_bytes=100663296 total_nvlink_bytes=471881688 weight_to_token_ratio=0.271170 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.177431692e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58828 payload_rows_remote=51162 dup_rows=6708 dup_rows_remote=5384 dedup_ratio=0.102356 router_tokens_per_expert=[0, 2, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 0, 178, 4065, 4094, 140, 0, 2742, 0, 0, 0, 893, 3873, 0, 0, 1, 3601, 0, 14, 0, 0, 165, 64, 2350, 0, 0, 7, 0, 0, 0, 0, 0, 3867, 0, 4, 56, 0, 4, 6, 0, 2, 0, 0, 3029, 0, 0, 1974, 0, 6] payload_rows_per_dst=[8134, 7826, 6142, 7666, 5160, 7748, 8186, 7966] dup_rows_per_dst=[300, 2268, 650, 1324, 26, 0, 88, 2052] combine_rows_per_src=[8134, 7826, 6142, 7666, 5160, 7748, 8186, 7966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54918 payload_rows_remote=46834 dup_rows=10618 dup_rows_remote=7896 dedup_ratio=0.162018 router_tokens_per_expert=[7, 2, 1755, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 32, 41, 4091, 4036, 126, 77, 0, 23, 0, 0, 3068, 4055, 0, 0, 2, 86, 0, 0, 6, 1, 1929, 0, 1672, 0, 0, 0, 47, 0, 39, 0, 0, 3984, 0, 0, 0, 0, 0, 7, 0, 0, 4, 2, 4046, 0, 0, 0, 0, 0] payload_rows_per_dst=[3766, 7188, 8192, 6980, 4418, 8084, 8190, 8100] dup_rows_per_dst=[1042, 0, 478, 6234, 132, 2722, 6, 4] combine_rows_per_src=[3766, 7188, 8192, 6980, 4418, 8084, 8190, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59186 payload_rows_remote=56498 dup_rows=6350 dup_rows_remote=6170 dedup_ratio=0.096893 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 1, 0, 0, 0, 29, 2002, 280, 2, 4036, 4020, 0, 1099, 53, 0, 32, 0, 4045, 4068, 1, 0, 0, 0, 0, 0, 3203, 0, 197, 228, 1003, 6, 0, 1, 24, 0, 5, 0, 17, 4044, 0, 0, 8, 0, 65, 1, 0, 0, 0, 0, 4094, 0, 13, 0, 1, 0] payload_rows_per_dst=[8136, 8192, 2688, 8148, 7504, 8154, 8174, 8190] dup_rows_per_dst=[0, 4292, 180, 8, 1770, 28, 46, 26] combine_rows_per_src=[8136, 8192, 2688, 8148, 7504, 8154, 8174, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=169287680 dispatch_meta_bytes=230640 combine_payload_bytes=169287680 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=169518320 ingress_bytes=219619328 link_bytes=219619328 token_bytes=338806000 weight_movement_bytes=50331648 total_nvlink_bytes=389137648 weight_to_token_ratio=0.148556 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.671049330e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=164036608 dispatch_meta_bytes=225984 combine_payload_bytes=164036608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=164262592 ingress_bytes=189202432 link_bytes=189202432 token_bytes=328299200 weight_movement_bytes=25165824 total_nvlink_bytes=353465024 weight_to_token_ratio=0.076655 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.439611896e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=154984448 dispatch_meta_bytes=196720 combine_payload_bytes=154984448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=155181168 ingress_bytes=180150272 link_bytes=180150272 token_bytes=310165616 weight_movement_bytes=25165824 total_nvlink_bytes=335331440 weight_to_token_ratio=0.081137 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.370735418e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48738 payload_rows_remote=40546 dup_rows=16798 dup_rows_remote=10892 dedup_ratio=0.256317 router_tokens_per_expert=[70, 0, 0, 0, 0, 1, 2909, 0, 1, 0, 0, 0, 0, 1, 0, 594, 0, 0, 4091, 3790, 0, 383, 503, 8, 0, 0, 154, 4069, 0, 0, 0, 1, 0, 3, 0, 0, 3591, 2, 4068, 1, 2, 0, 0, 11, 302, 0, 0, 4015, 0, 0, 0, 0, 12, 0, 0, 0, 401, 0, 3684, 0, 0, 101, 0, 0] payload_rows_per_dst=[8192, 7614, 8192, 1332, 7192, 8162, 24, 8030] dup_rows_per_dst=[5906, 1158, 8808, 84, 2, 498, 0, 342] combine_rows_per_src=[8192, 7614, 8192, 1332, 7192, 8162, 24, 8030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=231415808 dispatch_meta_bytes=250672 combine_payload_bytes=231415808 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=231666480 ingress_bytes=306913280 link_bytes=306913280 token_bytes=463082288 weight_movement_bytes=75497472 total_nvlink_bytes=538579760 weight_to_token_ratio=0.163033 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.335255442e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=209559552 dispatch_meta_bytes=226184 combine_payload_bytes=209559552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=209785736 ingress_bytes=259891200 link_bytes=259891200 token_bytes=419345288 weight_movement_bytes=50331648 total_nvlink_bytes=469676936 weight_to_token_ratio=0.120024 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.977471744e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=191832064 dispatch_meta_bytes=218920 combine_payload_bytes=191832064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=192050984 ingress_bytes=216997888 link_bytes=216997888 token_bytes=383883048 weight_movement_bytes=25165824 total_nvlink_bytes=409048872 weight_to_token_ratio=0.065556 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.651103200e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57888 payload_rows_remote=52910 dup_rows=7648 dup_rows_remote=6642 dedup_ratio=0.116699 router_tokens_per_expert=[1, 2, 0, 4093, 0, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 4084, 4, 0, 229, 0, 2, 0, 1, 112, 0, 0, 388, 0, 0, 0, 0, 0, 8, 0, 1, 823, 0, 0, 4096, 25, 0, 1, 0, 0, 4094, 0, 3054, 0, 0, 0, 18] payload_rows_per_dst=[7212, 8072, 8182, 8176, 4978, 8186, 4892, 8190] dup_rows_per_dst=[0, 0, 4, 458, 1006, 18, 20, 6142] combine_rows_per_src=[7212, 8072, 8182, 8176, 4978, 8186, 4892, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=166076416 dispatch_meta_bytes=205752 combine_payload_bytes=166076416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166282168 ingress_bytes=191242240 link_bytes=191242240 token_bytes=332358584 weight_movement_bytes=25165824 total_nvlink_bytes=357524408 weight_to_token_ratio=0.075719 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.455132478e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47942 payload_rows_remote=39812 dup_rows=17594 dup_rows_remote=17450 dedup_ratio=0.268463 router_tokens_per_expert=[126, 0, 953, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 36, 775, 856, 0, 4008, 3884, 0, 2, 9, 0, 0, 0, 3999, 3375, 0, 7, 0, 0, 2, 0, 1525, 0, 202, 2150, 4076, 1, 0, 0, 8, 0, 0, 0, 122, 3388, 0, 105, 0, 24, 0, 0, 0, 0, 10, 0, 3030, 0, 0, 0, 0, 0] payload_rows_per_dst=[2128, 1578, 7952, 8012, 5882, 8180, 8130, 6080] dup_rows_per_dst=[30, 234, 1550, 6750, 1878, 1496, 144, 5512] combine_rows_per_src=[2128, 1578, 7952, 8012, 5882, 8180, 8130, 6080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=216719360 dispatch_meta_bytes=238208 combine_payload_bytes=216719360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=216957568 ingress_bytes=241885184 link_bytes=241885184 token_bytes=433676928 weight_movement_bytes=25165824 total_nvlink_bytes=458842752 weight_to_token_ratio=0.058029 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.840466767e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50390 payload_rows_remote=42198 dup_rows=15146 dup_rows_remote=7732 dedup_ratio=0.231110 router_tokens_per_expert=[0, 0, 0, 4044, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3384, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 429, 0, 926, 0, 0, 817, 0, 527, 0, 84, 9, 22, 237, 4095, 0, 0, 0, 602, 0, 0, 0, 0, 1750, 0, 3, 3451, 1199, 0, 0, 0, 0, 1326, 0, 3990, 7, 0, 1542, 5] payload_rows_per_dst=[742, 6778, 7886, 4134, 8192, 8192, 6274, 8192] dup_rows_per_dst=[0, 0, 0, 210, 1756, 1100, 4666, 7414] combine_rows_per_src=[742, 6778, 7886, 4134, 8192, 8192, 6274, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56216 payload_rows_remote=50278 dup_rows=9320 dup_rows_remote=9320 dedup_ratio=0.142212 router_tokens_per_expert=[0, 9, 0, 4085, 0, 103, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2969, 0, 0, 0, 742, 23, 10, 493, 2, 0, 118, 0, 4018, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 3996, 0, 86, 0, 0, 0, 0, 22, 0, 826, 0, 0, 2277, 3794, 0, 6, 0, 0, 3317, 0, 2907, 0, 0, 0, 0] payload_rows_per_dst=[7132, 5938, 7176, 8114, 8186, 6244, 7612, 5814] dup_rows_per_dst=[1036, 0, 1994, 1022, 4448, 40, 780, 0] combine_rows_per_src=[7132, 5938, 7176, 8114, 8186, 6244, 7612, 5814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56902 payload_rows_remote=50718 dup_rows=8634 dup_rows_remote=8594 dedup_ratio=0.131744 router_tokens_per_expert=[0, 0, 0, 4065, 0, 0, 3841, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4096, 8, 0, 4011, 0, 29, 0, 0, 61, 10, 75, 0, 2, 3, 0, 2961, 1, 108, 64, 0, 388, 0, 0, 39, 0, 0, 0, 0, 0, 0, 1, 3873, 37, 0, 0, 3782, 3547, 0, 0, 0, 0, 593, 0, 1071, 0, 0, 99, 0] payload_rows_per_dst=[7682, 8192, 8044, 6184, 8180, 8192, 7102, 3326] dup_rows_per_dst=[0, 6, 52, 40, 584, 7686, 66, 200] combine_rows_per_src=[7682, 8192, 8044, 6184, 8180, 8192, 7102, 3326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56434 payload_rows_remote=48542 dup_rows=9102 dup_rows_remote=7076 dedup_ratio=0.138885 router_tokens_per_expert=[0, 0, 0, 4079, 1, 1, 4063, 0, 0, 11, 0, 0, 3, 80, 0, 0, 4081, 4, 0, 3926, 1, 0, 8, 0, 23, 0, 0, 7, 1026, 0, 0, 4066, 7, 0, 0, 0, 0, 0, 248, 14, 5, 0, 0, 0, 38, 0, 0, 237, 0, 0, 0, 4075, 0, 0, 251, 0, 1, 3017, 0, 3413, 26, 0, 0, 56] payload_rows_per_dst=[8130, 8190, 7892, 8192, 8176, 8192, 502, 7160] dup_rows_per_dst=[0, 160, 2026, 2052, 512, 526, 0, 3826] combine_rows_per_src=[8130, 8190, 7892, 8192, 8176, 8192, 502, 7160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53842 payload_rows_remote=45698 dup_rows=11694 dup_rows_remote=11694 dedup_ratio=0.178436 router_tokens_per_expert=[0, 0, 0, 4072, 35, 0, 3932, 0, 3, 0, 0, 244, 0, 0, 0, 0, 3740, 0, 0, 2519, 16, 0, 0, 0, 0, 48, 1384, 1, 3804, 130, 0, 446, 0, 0, 0, 0, 22, 0, 5, 114, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1, 26, 3989, 36, 0, 0, 18, 0, 3554, 572, 5, 2, 0, 0, 0] payload_rows_per_dst=[7934, 7970, 5046, 8184, 254, 8144, 8186, 8124] dup_rows_per_dst=[0, 4, 24, 3442, 28, 0, 8054, 142] combine_rows_per_src=[7934, 7970, 5046, 8184, 254, 8144, 8186, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=163069952 dispatch_meta_bytes=229048 combine_payload_bytes=163069952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=163299000 ingress_bytes=188235776 link_bytes=188235776 token_bytes=326368952 weight_movement_bytes=25165824 total_nvlink_bytes=351534776 weight_to_token_ratio=0.077109 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=28416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.432256761e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=172843008 dispatch_meta_bytes=199720 combine_payload_bytes=172843008 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=173042728 ingress_bytes=223174656 link_bytes=223174656 token_bytes=345885736 weight_movement_bytes=50331648 total_nvlink_bytes=396217384 weight_to_token_ratio=0.145515 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.698101268e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=39414 payload_rows_remote=31232 dup_rows=26122 dup_rows_remote=14960 dedup_ratio=0.398590 router_tokens_per_expert=[0, 0, 0, 4077, 0, 0, 3505, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4059, 1, 0, 4094, 3, 0, 0, 0, 0, 0, 2098, 0, 4086, 9, 1, 2, 0, 1, 0, 0, 3022, 0, 0, 93, 0, 0, 0, 2, 0, 0, 0, 526, 38, 0, 0, 4092, 78, 0, 0, 0, 0, 2492, 0, 467, 0, 0, 19, 1] payload_rows_per_dst=[8182, 8192, 5456, 7290, 8192, 1052, 76, 974] dup_rows_per_dst=[11162, 3900, 3754, 1078, 6224, 4, 0, 0] combine_rows_per_src=[8182, 8192, 5456, 7290, 8192, 1052, 76, 974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=205938688 dispatch_meta_bytes=238392 combine_payload_bytes=205938688 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=206177080 ingress_bytes=256270336 link_bytes=256270336 token_bytes=412115768 weight_movement_bytes=50331648 total_nvlink_bytes=462447416 weight_to_token_ratio=0.122130 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.949921153e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=207740928 dispatch_meta_bytes=237248 combine_payload_bytes=207740928 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=207978176 ingress_bytes=258072576 link_bytes=258072576 token_bytes=415719104 weight_movement_bytes=50331648 total_nvlink_bytes=466050752 weight_to_token_ratio=0.121071 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.963634117e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=198828032 dispatch_meta_bytes=222472 combine_payload_bytes=198828032 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=199050504 ingress_bytes=274325504 link_bytes=274325504 token_bytes=397878536 weight_movement_bytes=75497472 total_nvlink_bytes=473376008 weight_to_token_ratio=0.189750 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=24576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.087300120e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=187179008 dispatch_meta_bytes=229568 combine_payload_bytes=187179008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=187408576 ingress_bytes=212344832 link_bytes=212344832 token_bytes=374587584 weight_movement_bytes=25165824 total_nvlink_bytes=399753408 weight_to_token_ratio=0.067183 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=27392 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.615698821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49880 payload_rows_remote=41812 dup_rows=15656 dup_rows_remote=12618 dedup_ratio=0.238892 router_tokens_per_expert=[0, 101, 110, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 4095, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 1023, 0, 0, 0, 0, 10, 0, 0, 0, 0, 2071, 3482, 0, 0, 373, 693, 18, 8, 1, 0, 4044, 0, 0, 40, 0, 7, 0, 0, 0, 4089, 0, 56, 0, 0, 0, 0, 0, 0, 3869] payload_rows_per_dst=[1138, 1552, 8174, 6746, 8068, 8174, 8192, 7836] dup_rows_per_dst=[250, 20, 4, 1958, 3038, 2100, 3176, 5110] combine_rows_per_src=[1138, 1552, 8174, 6746, 8068, 8174, 8192, 7836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=127926272 dispatch_meta_bytes=184768 combine_payload_bytes=127926272 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=128111040 ingress_bytes=178257920 link_bytes=178257920 token_bytes=256037312 weight_movement_bytes=50331648 total_nvlink_bytes=306368960 weight_to_token_ratio=0.196579 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.356336806e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=171261952 dispatch_meta_bytes=217720 combine_payload_bytes=171261952 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=171479672 ingress_bytes=221593600 link_bytes=221593600 token_bytes=342741624 weight_movement_bytes=50331648 total_nvlink_bytes=393073272 weight_to_token_ratio=0.146850 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.686071259e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54860 payload_rows_remote=46668 dup_rows=10676 dup_rows_remote=1562 dedup_ratio=0.162903 router_tokens_per_expert=[0, 0, 0, 3968, 1, 0, 3160, 1, 0, 0, 35, 0, 0, 0, 0, 5, 4027, 8, 0, 3947, 0, 0, 0, 45, 490, 0, 0, 0, 2433, 0, 15, 542, 8, 13, 0, 1, 0, 2, 588, 1798, 0, 0, 3, 0, 1, 0, 8, 0, 934, 0, 3, 4096, 3620, 0, 0, 0, 0, 0, 0, 2862, 1, 6, 147, 0] payload_rows_per_dst=[6324, 8088, 7988, 6000, 4282, 7958, 8192, 6028] dup_rows_per_dst=[0, 46, 12, 960, 538, 2, 9114, 4] combine_rows_per_src=[6324, 8088, 7988, 6000, 4282, 7958, 8192, 6028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48522 payload_rows_remote=40336 dup_rows=17014 dup_rows_remote=16484 dedup_ratio=0.259613 router_tokens_per_expert=[0, 0, 448, 7, 0, 3965, 144, 0, 2, 0, 1, 0, 123, 3953, 0, 3811, 0, 12, 5, 0, 101, 1, 3956, 1, 0, 3556, 0, 0, 0, 30, 110, 0, 0, 0, 0, 2089, 4027, 0, 0, 0, 0, 167, 0, 0, 3104, 2719, 0, 0, 4, 0, 0, 0, 23, 0, 4, 0, 331, 1, 0, 0, 0, 56, 0, 17] payload_rows_per_dst=[8192, 250, 8098, 8186, 8070, 7482, 58, 8186] dup_rows_per_dst=[936, 2, 54, 6828, 4162, 4498, 4, 530] combine_rows_per_src=[8192, 250, 8098, 8186, 8070, 7482, 58, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45446 payload_rows_remote=45416 dup_rows=20090 dup_rows_remote=20080 dedup_ratio=0.306549 router_tokens_per_expert=[6, 0, 0, 0, 0, 3006, 0, 0, 0, 0, 0, 70, 6, 4027, 1, 4015, 0, 0, 0, 0, 0, 2, 1988, 4, 0, 0, 6, 14, 0, 0, 0, 0, 0, 23, 0, 232, 27, 0, 0, 3984, 0, 0, 1622, 0, 0, 4082, 7, 0, 0, 0, 64, 0, 0, 9, 4096, 0, 1630, 0, 0, 2, 0, 0, 0, 3845] payload_rows_per_dst=[7076, 8172, 3988, 30, 4250, 5836, 8192, 7902] dup_rows_per_dst=[3558, 6204, 0, 10, 562, 4696, 2008, 3052] combine_rows_per_src=[7076, 8172, 3988, 30, 4250, 5836, 8192, 7902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45318 payload_rows_remote=37166 dup_rows=20218 dup_rows_remote=12900 dedup_ratio=0.308502 router_tokens_per_expert=[554, 101, 0, 7, 0, 14, 2, 0, 0, 0, 13, 0, 173, 3520, 1, 4028, 0, 0, 0, 22, 0, 0, 1854, 837, 0, 3853, 0, 0, 0, 1, 1101, 0, 0, 0, 70, 2806, 574, 0, 0, 4084, 30, 0, 3, 0, 3, 3871, 28, 0, 23, 6, 1211, 0, 0, 14, 3919, 0, 0, 33, 0, 0, 0, 0, 12, 0] payload_rows_per_dst=[8166, 8152, 5730, 7892, 8190, 98, 7006, 84] dup_rows_per_dst=[932, 7318, 1066, 2018, 6878, 30, 1970, 6] combine_rows_per_src=[8166, 8152, 5730, 7892, 8190, 98, 7006, 84] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53352 payload_rows_remote=45166 dup_rows=12184 dup_rows_remote=12184 dedup_ratio=0.185913 router_tokens_per_expert=[0, 7, 1, 0, 6, 153, 207, 0, 0, 2, 0, 0, 6, 3877, 103, 3868, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 2440, 4, 0, 0, 0, 3, 0, 0, 0, 35, 3237, 163, 1, 2, 1063, 0, 0, 295, 4, 0, 3911, 1, 0, 18, 0, 964, 31, 0, 191, 4077, 0, 0, 0, 0, 0, 61, 0, 0, 3942] payload_rows_per_dst=[8136, 8134, 8186, 4892, 7212, 598, 8188, 8006] dup_rows_per_dst=[434, 7578, 0, 6, 1790, 2, 2374, 0] combine_rows_per_src=[8136, 8134, 8186, 4892, 7212, 598, 8188, 8006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48968 payload_rows_remote=40788 dup_rows=16568 dup_rows_remote=7832 dedup_ratio=0.252808 router_tokens_per_expert=[55, 3380, 0, 0, 0, 2409, 0, 0, 0, 24, 0, 0, 0, 4091, 0, 4075, 0, 1, 0, 0, 0, 0, 4034, 0, 0, 1372, 447, 0, 0, 260, 0, 0, 1, 0, 0, 30, 0, 1, 0, 3512, 610, 0, 202, 0, 48, 4085, 1, 0, 0, 0, 0, 0, 2, 12, 4011, 0, 31, 0, 0, 0, 0, 40, 0, 34] payload_rows_per_dst=[8180, 48, 8070, 8192, 64, 8180, 8046, 8188] dup_rows_per_dst=[3508, 0, 0, 4116, 0, 8736, 4, 204] combine_rows_per_src=[8180, 48, 8070, 8192, 64, 8180, 8046, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=165216256 dispatch_meta_bytes=227280 combine_payload_bytes=165216256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=165443536 ingress_bytes=190382080 link_bytes=190382080 token_bytes=330659792 weight_movement_bytes=25165824 total_nvlink_bytes=355825616 weight_to_token_ratio=0.076108 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.448587654e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=191152128 dispatch_meta_bytes=192920 combine_payload_bytes=191152128 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=191345048 ingress_bytes=266649600 link_bytes=266649600 token_bytes=382497176 weight_movement_bytes=75497472 total_nvlink_bytes=457994648 weight_to_token_ratio=0.197380 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=12288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.028895359e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44056 payload_rows_remote=37412 dup_rows=21480 dup_rows_remote=21480 dedup_ratio=0.327759 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 21, 0, 0, 2, 4096, 3, 4067, 0, 0, 0, 0, 1, 2, 4065, 13, 0, 441, 0, 0, 0, 0, 2725, 0, 0, 0, 0, 0, 3085, 0, 0, 1990, 367, 0, 1, 0, 12, 3321, 4, 0, 0, 0, 407, 0, 0, 0, 4024, 0, 16, 0, 3, 0, 0, 6, 0, 4095] payload_rows_per_dst=[6644, 8192, 8168, 5464, 5826, 756, 814, 8192] dup_rows_per_dst=[0, 9748, 8042, 868, 2762, 12, 0, 48] combine_rows_per_src=[6644, 8192, 8168, 5464, 5826, 756, 814, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=186023936 dispatch_meta_bytes=261984 combine_payload_bytes=186023936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186285920 ingress_bytes=211189760 link_bytes=211189760 token_bytes=372309856 weight_movement_bytes=25165824 total_nvlink_bytes=397475680 weight_to_token_ratio=0.067594 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.606910057e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=152231936 dispatch_meta_bytes=200264 combine_payload_bytes=152231936 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=152432200 ingress_bytes=252895232 link_bytes=252895232 token_bytes=304664136 weight_movement_bytes=100663296 total_nvlink_bytes=405327432 weight_to_token_ratio=0.330407 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.924240511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56266 payload_rows_remote=48074 dup_rows=9270 dup_rows_remote=8844 dedup_ratio=0.141449 router_tokens_per_expert=[0, 2488, 0, 0, 221, 0, 411, 0, 0, 4071, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 2103, 0, 34, 4096, 0, 1, 0, 178, 0, 0, 121, 0, 3225, 3, 0, 0, 1, 21, 3, 0, 0, 0, 0, 0, 0, 29, 0, 1, 0, 0, 3964, 3854, 0, 3840] payload_rows_per_dst=[1258, 8152, 8142, 8188, 8192, 6696, 7956, 7682] dup_rows_per_dst=[6, 56, 4974, 3726, 426, 46, 36, 0] combine_rows_per_src=[1258, 8152, 8142, 8188, 8192, 6696, 7956, 7682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=184999936 dispatch_meta_bytes=229400 combine_payload_bytes=184999936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=185229336 ingress_bytes=210165760 link_bytes=210165760 token_bytes=370229272 weight_movement_bytes=25165824 total_nvlink_bytes=395395096 weight_to_token_ratio=0.067974 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599118600e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=167067648 dispatch_meta_bytes=194480 combine_payload_bytes=167067648 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=167262128 ingress_bytes=217399296 link_bytes=217399296 token_bytes=334329776 weight_movement_bytes=50331648 total_nvlink_bytes=384661424 weight_to_token_ratio=0.150545 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.654157452e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=153239552 dispatch_meta_bytes=235568 combine_payload_bytes=153239552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=153475120 ingress_bytes=178405376 link_bytes=178405376 token_bytes=306714672 weight_movement_bytes=25165824 total_nvlink_bytes=331880496 weight_to_token_ratio=0.082050 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.357458775e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=196911104 dispatch_meta_bytes=227672 combine_payload_bytes=196911104 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=197138776 ingress_bytes=247242752 link_bytes=247242752 token_bytes=394049880 weight_movement_bytes=50331648 total_nvlink_bytes=444381528 weight_to_token_ratio=0.127729 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.881231670e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47808 payload_rows_remote=39618 dup_rows=17728 dup_rows_remote=6384 dedup_ratio=0.270508 router_tokens_per_expert=[0, 4094, 24, 0, 3234, 0, 1264, 484, 0, 3537, 0, 6, 0, 0, 0, 0, 0, 26, 1, 52, 0, 0, 1643, 0, 1, 0, 2, 0, 3, 378, 0, 0, 0, 3284, 0, 0, 0, 367, 1422, 0, 0, 0, 146, 2, 2134, 0, 0, 0, 0, 0, 105, 0, 0, 39, 0, 757, 0, 1, 0, 1, 4079, 4090, 0, 1592] payload_rows_per_dst=[6492, 7086, 3426, 8190, 8154, 4468, 1802, 8190] dup_rows_per_dst=[3520, 0, 18, 758, 1992, 96, 0, 11344] combine_rows_per_src=[6492, 7086, 3426, 8190, 8154, 4468, 1802, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51304 payload_rows_remote=43112 dup_rows=14232 dup_rows_remote=6920 dedup_ratio=0.217163 router_tokens_per_expert=[16, 0, 0, 3, 109, 4061, 4, 0, 3, 0, 43, 36, 0, 4046, 1, 4049, 0, 0, 461, 0, 0, 0, 2517, 0, 0, 5, 0, 0, 0, 111, 0, 0, 0, 0, 1, 175, 2156, 0, 0, 5, 0, 0, 167, 0, 3193, 3803, 1, 0, 0, 0, 3654, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 1, 1, 46] payload_rows_per_dst=[8156, 94, 5956, 8172, 4642, 7992, 8192, 8100] dup_rows_per_dst=[230, 72, 0, 158, 22, 6346, 7312, 92] combine_rows_per_src=[8156, 94, 5956, 8172, 4642, 7992, 8192, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58984 payload_rows_remote=53850 dup_rows=6552 dup_rows_remote=6454 dedup_ratio=0.099976 router_tokens_per_expert=[0, 4010, 0, 45, 0, 0, 9, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 4096, 0, 55, 0, 0, 0, 91, 59, 0, 0, 65, 3928, 0, 0, 0, 4, 6, 0, 0, 0, 4086, 0, 1506, 0, 0, 0, 112, 0, 167, 0, 0, 0, 0, 80, 0, 0, 23, 0, 3687, 2411, 40, 3737] payload_rows_per_dst=[5920, 8192, 8192, 5134, 7908, 8192, 7846, 7600] dup_rows_per_dst=[32, 0, 3086, 98, 98, 2992, 246, 0] combine_rows_per_src=[5920, 8192, 8192, 5134, 7908, 8192, 7846, 7600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51810 payload_rows_remote=43650 dup_rows=13726 dup_rows_remote=13726 dedup_ratio=0.209442 router_tokens_per_expert=[0, 4089, 0, 0, 1035, 2, 1561, 0, 0, 4052, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 2446, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 46, 2, 0, 800, 9, 0, 0, 0, 4093, 2, 956, 7, 31, 2524, 0, 33, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 1046, 2, 2515] payload_rows_per_dst=[8192, 8160, 4922, 3684, 7190, 8192, 8146, 3324] dup_rows_per_dst=[5182, 0, 0, 130, 1316, 7034, 64, 0] combine_rows_per_src=[8192, 8160, 4922, 3684, 7190, 8192, 8146, 3324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52638 payload_rows_remote=45918 dup_rows=12898 dup_rows_remote=12832 dedup_ratio=0.196808 router_tokens_per_expert=[0, 4010, 197, 0, 1727, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 40, 3353, 0, 0, 0, 0, 0, 0, 2655, 0, 0, 1672, 4072, 0, 0, 0, 72, 0, 0, 794, 12, 920, 20, 11, 1417, 0, 0, 0, 184, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 2413, 132, 847] payload_rows_per_dst=[8056, 8188, 6720, 6642, 8192, 4906, 8120, 1814] dup_rows_per_dst=[3812, 2, 66, 3494, 3440, 1442, 498, 144] combine_rows_per_src=[8056, 8188, 6720, 6642, 8192, 4906, 8120, 1814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=162275328 dispatch_meta_bytes=184008 combine_payload_bytes=162275328 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=162459336 ingress_bytes=262938624 link_bytes=262938624 token_bytes=324734664 weight_movement_bytes=100663296 total_nvlink_bytes=425397960 weight_to_token_ratio=0.309986 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.000659120e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56098 payload_rows_remote=47938 dup_rows=9438 dup_rows_remote=8748 dedup_ratio=0.144012 router_tokens_per_expert=[0, 1546, 0, 3, 0, 0, 362, 0, 0, 4072, 0, 0, 0, 0, 2, 0, 0, 14, 0, 0, 0, 0, 3607, 0, 0, 0, 0, 6, 1, 0, 1111, 0, 15, 3925, 1, 0, 7, 0, 0, 1, 0, 0, 3522, 0, 706, 0, 0, 197, 0, 0, 775, 0, 0, 0, 0, 1313, 0, 0, 20, 0, 4095, 3462, 0, 4005] payload_rows_per_dst=[728, 8148, 7486, 7454, 7882, 8160, 8190, 8050] dup_rows_per_dst=[2, 0, 2848, 1706, 16, 690, 4176, 0] combine_rows_per_src=[728, 8148, 7486, 7454, 7882, 8160, 8190, 8050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=176586752 dispatch_meta_bytes=200128 combine_payload_bytes=176586752 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=176786880 ingress_bytes=226918400 link_bytes=226918400 token_bytes=353373632 weight_movement_bytes=50331648 total_nvlink_bytes=403705280 weight_to_token_ratio=0.142432 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.726586834e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49380 payload_rows_remote=41192 dup_rows=16156 dup_rows_remote=10120 dedup_ratio=0.246521 router_tokens_per_expert=[0, 4089, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4094, 0, 27, 0, 15, 4, 0, 1, 0, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3933, 8, 0, 0, 4, 4069, 4071, 7, 471] payload_rows_per_dst=[8188, 8192, 8192, 8190, 2442, 5952, 8192, 32] dup_rows_per_dst=[6036, 1388, 0, 896, 14, 0, 7816, 6] combine_rows_per_src=[8188, 8192, 8192, 8190, 2442, 5952, 8192, 32] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49410 payload_rows_remote=41222 dup_rows=16126 dup_rows_remote=16124 dedup_ratio=0.246063 router_tokens_per_expert=[0, 0, 123, 0, 4079, 0, 0, 0, 0, 4093, 0, 0, 0, 2810, 0, 0, 534, 2918, 0, 0, 0, 3706, 1325, 0, 0, 0, 25, 0, 0, 0, 1, 1, 4094, 0, 0, 1, 0, 0, 0, 0, 2, 0, 4096, 3578, 0, 0, 0, 0, 0, 0, 30, 0, 26, 0, 0, 0, 4, 0, 0, 280, 1, 0, 0, 1041] payload_rows_per_dst=[246, 5620, 8150, 8192, 8188, 8192, 8184, 2638] dup_rows_per_dst=[0, 0, 8816, 48, 2, 7160, 86, 14] combine_rows_per_src=[246, 5620, 8150, 8192, 8188, 8192, 8184, 2638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=220569600 dispatch_meta_bytes=241216 combine_payload_bytes=220569600 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=220810816 ingress_bytes=270901248 link_bytes=270901248 token_bytes=441380416 weight_movement_bytes=50331648 total_nvlink_bytes=491712064 weight_to_token_ratio=0.114032 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.061245488e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=178790400 dispatch_meta_bytes=229504 combine_payload_bytes=178790400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=179019904 ingress_bytes=203956224 link_bytes=203956224 token_bytes=357810304 weight_movement_bytes=25165824 total_nvlink_bytes=382976128 weight_to_token_ratio=0.070333 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.551871206e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=188080128 dispatch_meta_bytes=235000 combine_payload_bytes=188080128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188315128 ingress_bytes=213245952 link_bytes=213245952 token_bytes=376395256 weight_movement_bytes=25165824 total_nvlink_bytes=401561080 weight_to_token_ratio=0.066860 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.622555303e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=196354048 dispatch_meta_bytes=226744 combine_payload_bytes=196354048 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=196580792 ingress_bytes=246685696 link_bytes=246685696 token_bytes=392934840 weight_movement_bytes=50331648 total_nvlink_bytes=443266488 weight_to_token_ratio=0.128092 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22400 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.876993117e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=168722432 dispatch_meta_bytes=205248 combine_payload_bytes=168722432 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=168927680 ingress_bytes=244219904 link_bytes=244219904 token_bytes=337650112 weight_movement_bytes=75497472 total_nvlink_bytes=413147584 weight_to_token_ratio=0.223597 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.858231289e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=168845312 dispatch_meta_bytes=229384 combine_payload_bytes=168845312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=169074696 ingress_bytes=194011136 link_bytes=194011136 token_bytes=337920008 weight_movement_bytes=25165824 total_nvlink_bytes=363085832 weight_to_token_ratio=0.074473 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.476200577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43340 payload_rows_remote=35292 dup_rows=22196 dup_rows_remote=13916 dedup_ratio=0.338684 router_tokens_per_expert=[0, 1, 329, 0, 4092, 5, 0, 0, 128, 2290, 54, 0, 18, 0, 0, 5, 2504, 0, 0, 39, 11, 4084, 229, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 4072, 0, 0, 0, 0, 7, 0, 6, 0, 0, 3370, 398, 0, 0, 0, 69, 643, 0, 0, 0, 0, 0, 0, 1609, 0, 176, 0, 4024, 0, 0, 0, 594] payload_rows_per_dst=[670, 258, 7278, 8058, 8160, 2682, 8186, 8048] dup_rows_per_dst=[0, 152, 3540, 4544, 1118, 60, 4502, 8280] combine_rows_per_src=[670, 258, 7278, 8058, 8160, 2682, 8186, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52960 payload_rows_remote=50544 dup_rows=12576 dup_rows_remote=12544 dedup_ratio=0.191895 router_tokens_per_expert=[0, 4069, 905, 0, 2302, 0, 848, 1, 2, 3984, 0, 0, 0, 0, 0, 0, 0, 31, 9, 0, 0, 1, 1715, 3, 0, 24, 0, 0, 1, 0, 2174, 6, 67, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 3334, 0, 0, 1, 0, 0, 0, 0, 0, 22, 0, 78, 0, 0, 0, 0, 0, 26, 3598, 1418, 0, 4079] payload_rows_per_dst=[5400, 7972, 7368, 6808, 8140, 6670, 2416, 8186] dup_rows_per_dst=[2712, 0, 96, 438, 134, 0, 32, 9164] combine_rows_per_src=[5400, 7972, 7368, 6808, 8140, 6670, 2416, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46436 payload_rows_remote=38300 dup_rows=19100 dup_rows_remote=16442 dedup_ratio=0.291443 router_tokens_per_expert=[0, 6, 0, 19, 3255, 0, 382, 0, 0, 4006, 0, 2, 0, 3651, 0, 0, 17, 0, 1, 3, 0, 1, 1796, 0, 0, 2, 0, 0, 1, 6, 1381, 1, 3693, 0, 46, 0, 0, 0, 0, 19, 3040, 0, 4059, 3861, 76, 0, 0, 0, 0, 9, 1001, 18, 0, 0, 0, 901, 0, 0, 0, 839, 0, 0, 0, 676] payload_rows_per_dst=[780, 7306, 3636, 8136, 7482, 8186, 8184, 2726] dup_rows_per_dst=[34, 0, 0, 2658, 34, 13886, 2184, 304] combine_rows_per_src=[780, 7306, 3636, 8136, 7482, 8186, 8184, 2726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52184 payload_rows_remote=45360 dup_rows=13352 dup_rows_remote=13352 dedup_ratio=0.203735 router_tokens_per_expert=[260, 5, 4065, 0, 3750, 0, 0, 116, 0, 3406, 0, 0, 0, 0, 6, 0, 13, 3, 0, 202, 0, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4070, 0, 93, 0, 0, 2, 0, 1, 0, 0, 3937, 0, 4, 0, 0, 0, 0, 726, 4037, 305, 0, 0, 81, 174, 0, 3940, 0, 28, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6824, 4938, 46, 8178, 7882, 8190, 7934] dup_rows_per_dst=[10718, 0, 22, 0, 154, 0, 2456, 2] combine_rows_per_src=[8192, 6824, 4938, 46, 8178, 7882, 8190, 7934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54450 payload_rows_remote=46384 dup_rows=11086 dup_rows_remote=9166 dedup_ratio=0.169159 router_tokens_per_expert=[0, 0, 0, 72, 4090, 212, 51, 0, 1, 4094, 0, 0, 5, 4047, 0, 0, 0, 0, 0, 2, 1, 982, 4008, 0, 17, 0, 7, 53, 0, 0, 3535, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 7, 2, 4037, 0, 0, 0, 0, 0, 0, 395, 12, 0, 0, 0, 74, 0, 1, 263, 0, 211, 0, 0, 0, 2582] payload_rows_per_dst=[7048, 8192, 8066, 7178, 8014, 8088, 1948, 5916] dup_rows_per_dst=[670, 7956, 1920, 192, 0, 4, 146, 198] combine_rows_per_src=[7048, 8192, 8066, 7178, 8014, 8088, 1948, 5916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=144556032 dispatch_meta_bytes=196832 combine_payload_bytes=144556032 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=144752864 ingress_bytes=169721856 link_bytes=169721856 token_bytes=289308896 weight_movement_bytes=25165824 total_nvlink_bytes=314474720 weight_to_token_ratio=0.086986 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.291387221e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=207028224 dispatch_meta_bytes=252352 combine_payload_bytes=207028224 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=207280576 ingress_bytes=232194048 link_bytes=232194048 token_bytes=414308800 weight_movement_bytes=25165824 total_nvlink_bytes=439474624 weight_to_token_ratio=0.060742 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.766728420e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57370 payload_rows_remote=49222 dup_rows=8166 dup_rows_remote=8152 dedup_ratio=0.124603 router_tokens_per_expert=[0, 21, 0, 4093, 4050, 0, 0, 0, 4096, 4, 157, 0, 4011, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4062, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8130, 314, 8022, 8192, 8148, 8186, 8186, 8192] dup_rows_per_dst=[12, 8, 2, 8126, 14, 0, 2, 2] combine_rows_per_src=[8130, 314, 8022, 8192, 8148, 8186, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60928 payload_rows_remote=55074 dup_rows=4608 dup_rows_remote=4608 dedup_ratio=0.070312 router_tokens_per_expert=[0, 1, 4094, 1, 4090, 0, 0, 0, 0, 3908, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 26, 3951, 1403, 0, 0, 0, 0, 0, 189, 0, 0, 0, 4084, 5, 1, 0, 2, 0, 6, 0, 0, 0, 4072, 0, 0, 0, 0, 4, 15, 112, 0, 0, 0, 0, 0, 3, 0, 2286, 0, 19, 0, 0, 3, 758] payload_rows_per_dst=[8190, 7470, 8162, 8060, 8188, 5854, 8190, 6814] dup_rows_per_dst=[2, 0, 2598, 134, 8, 0, 250, 1616] combine_rows_per_src=[8190, 7470, 8162, 8060, 8188, 5854, 8190, 6814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=41432 payload_rows_remote=33240 dup_rows=24104 dup_rows_remote=13624 dedup_ratio=0.367798 router_tokens_per_expert=[0, 0, 3156, 9, 4089, 14, 0, 0, 0, 4069, 0, 0, 0, 4067, 0, 0, 59, 0, 0, 0, 0, 2068, 2442, 0, 0, 0, 1160, 0, 4, 0, 19, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 12, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4978, 2330, 8172, 2712, 4, 6852] dup_rows_per_dst=[10480, 13564, 24, 36, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 4978, 2330, 8172, 2712, 4, 6852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=156876800 dispatch_meta_bytes=218968 combine_payload_bytes=156876800 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=157095768 ingress_bytes=182042624 link_bytes=182042624 token_bytes=313972568 weight_movement_bytes=25165824 total_nvlink_bytes=339138392 weight_to_token_ratio=0.080153 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.385134030e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=185794560 dispatch_meta_bytes=234848 combine_payload_bytes=185794560 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=186029408 ingress_bytes=236126208 link_bytes=236126208 token_bytes=371823968 weight_movement_bytes=50331648 total_nvlink_bytes=422155616 weight_to_token_ratio=0.135364 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.796647614e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=189988864 dispatch_meta_bytes=222200 combine_payload_bytes=189988864 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=190211064 ingress_bytes=265486336 link_bytes=265486336 token_bytes=380199928 weight_movement_bytes=75497472 total_nvlink_bytes=455697400 weight_to_token_ratio=0.198573 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.020044264e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=201613312 dispatch_meta_bytes=229496 combine_payload_bytes=201613312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201842808 ingress_bytes=226779136 link_bytes=226779136 token_bytes=403456120 weight_movement_bytes=25165824 total_nvlink_bytes=428621944 weight_to_token_ratio=0.062376 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=25984 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.725527196e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=136151040 dispatch_meta_bytes=187456 combine_payload_bytes=136151040 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=136338496 ingress_bytes=186482688 link_bytes=186482688 token_bytes=272489536 weight_movement_bytes=50331648 total_nvlink_bytes=322821184 weight_to_token_ratio=0.184710 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.055147 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.418917787e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=225583104 dispatch_meta_bytes=238728 combine_payload_bytes=225583104 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=225821832 ingress_bytes=301080576 link_bytes=301080576 token_bytes=451404936 weight_movement_bytes=75497472 total_nvlink_bytes=526902408 weight_to_token_ratio=0.167250 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=20480 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.290875304e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=36258 dup_rows=21086 dup_rows_remote=19186 dedup_ratio=0.321747 router_tokens_per_expert=[0, 35, 0, 4044, 3651, 0, 0, 18, 77, 624, 0, 0, 4042, 3316, 4049, 4096, 0, 26, 0, 0, 0, 0, 0, 1, 825, 0, 16, 1, 0, 0, 0, 0, 0, 47, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 3719, 133, 0, 1, 0, 0, 0, 0, 0, 8, 0, 18, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[80, 8132, 8138, 1822, 1788, 8192, 8106, 8192] dup_rows_per_dst=[26, 5988, 0, 16, 94, 13062, 0, 1900] combine_rows_per_src=[80, 8132, 8138, 1822, 1788, 8192, 8106, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55726 payload_rows_remote=47534 dup_rows=9810 dup_rows_remote=1072 dedup_ratio=0.149689 router_tokens_per_expert=[0, 0, 0, 4096, 4056, 0, 0, 0, 3756, 0, 3002, 49, 361, 346, 799, 4082, 0, 0, 1, 0, 2, 0, 0, 0, 602, 0, 7, 0, 3, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4067, 0, 3395, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8112, 7322, 728, 8192, 8134, 6876, 8192, 8170] dup_rows_per_dst=[0, 1070, 0, 8738, 0, 0, 0, 2] combine_rows_per_src=[8112, 7322, 728, 8192, 8134, 6876, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43990 payload_rows_remote=35798 dup_rows=21546 dup_rows_remote=16712 dedup_ratio=0.328766 router_tokens_per_expert=[1393, 3698, 9, 0, 4057, 0, 48, 9, 0, 4012, 0, 0, 0, 0, 0, 1, 0, 1379, 0, 0, 0, 36, 155, 134, 0, 0, 0, 0, 0, 0, 3947, 3, 3220, 0, 699, 0, 0, 0, 0, 0, 0, 0, 4070, 415, 0, 0, 0, 0, 0, 27, 2341, 24, 0, 0, 64, 0, 210, 964, 0, 95, 0, 2, 0, 1756] payload_rows_per_dst=[7584, 2, 3322, 8038, 7830, 830, 8192, 8192] dup_rows_per_dst=[2730, 0, 86, 7886, 8, 0, 4834, 6002] combine_rows_per_src=[7584, 2, 3322, 8038, 7830, 830, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55600 payload_rows_remote=47520 dup_rows=9936 dup_rows_remote=5368 dedup_ratio=0.151611 router_tokens_per_expert=[0, 0, 0, 4062, 3764, 0, 95, 820, 4040, 0, 3962, 0, 4096, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 9, 30, 0, 249, 0, 143, 0, 0, 0, 0, 0, 0, 1, 0, 13, 1, 0, 0, 0, 0, 0, 0, 3277, 0, 3864, 17, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 5] payload_rows_per_dst=[8190, 8080, 8192, 536, 6580, 7762, 8192, 8068] dup_rows_per_dst=[1452, 4568, 3150, 248, 4, 0, 514, 0] combine_rows_per_src=[8190, 8080, 8192, 536, 6580, 7762, 8192, 8068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52722 payload_rows_remote=44530 dup_rows=12814 dup_rows_remote=8158 dedup_ratio=0.195526 router_tokens_per_expert=[339, 0, 0, 4080, 3897, 0, 139, 0, 4094, 4066, 0, 0, 3734, 0, 0, 3948, 0, 0, 0, 0, 0, 327, 0, 0, 23, 0, 0, 158, 0, 0, 11, 0, 3, 0, 0, 3574, 0, 0, 38, 0, 0, 1, 0, 0, 4048, 0, 131, 0, 0, 15, 5, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 7, 0, 25] payload_rows_per_dst=[8066, 8132, 8192, 3778, 8190, 264, 8190, 7910] dup_rows_per_dst=[684, 0, 4656, 68, 7136, 0, 220, 50] combine_rows_per_src=[8066, 8132, 8192, 3778, 8190, 264, 8190, 7910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=148512768 dispatch_meta_bytes=221776 combine_payload_bytes=148512768 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=148734544 ingress_bytes=198844416 link_bytes=198844416 token_bytes=297247312 weight_movement_bytes=50331648 total_nvlink_bytes=347578960 weight_to_token_ratio=0.169326 gemm_rows_defined=33920 gemm_rows_charged=34816 gemm_tail_pad_rows=896 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025735 gemm_active_groups=11 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.512976254e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56966 payload_rows_remote=48774 dup_rows=8570 dup_rows_remote=8558 dedup_ratio=0.130768 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 3878, 0, 3058, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1842, 0, 3057, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 4544, 7764, 8192, 8192, 6116, 5776, 8190] dup_rows_per_dst=[0, 842, 44, 6116, 12, 1544, 12, 0] combine_rows_per_src=[8192, 4544, 7764, 8192, 8192, 6116, 5776, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51450 payload_rows_remote=43258 dup_rows=14086 dup_rows_remote=8374 dedup_ratio=0.214935 router_tokens_per_expert=[0, 4, 0, 4085, 4079, 0, 0, 0, 4095, 0, 1032, 3, 4024, 4027, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 26, 0, 0, 0, 0, 0, 115, 0, 0, 69, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 0, 3843, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150] payload_rows_per_dst=[8192, 8192, 8088, 230, 6364, 7686, 4506, 8192] dup_rows_per_dst=[5712, 8058, 14, 0, 2, 0, 0, 300] combine_rows_per_src=[8192, 8192, 8088, 230, 6364, 7686, 4506, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=194699264 dispatch_meta_bytes=194424 combine_payload_bytes=194699264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=194893688 ingress_bytes=219865088 link_bytes=219865088 token_bytes=389592952 weight_movement_bytes=25165824 total_nvlink_bytes=414758776 weight_to_token_ratio=0.064595 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.672919280e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60106 payload_rows_remote=52082 dup_rows=5430 dup_rows_remote=5412 dedup_ratio=0.082855 router_tokens_per_expert=[0, 0, 0, 4096, 4073, 0, 354, 0, 4084, 1254, 4, 0, 3526, 0, 380, 4082, 0, 0, 0, 1, 32, 0, 11, 555, 4, 0, 0, 0, 0, 1854, 0, 0, 177, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4086, 20, 3992, 9, 0, 4, 165, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3276, 7888, 8178, 8190, 8024, 8192, 8166] dup_rows_per_dst=[662, 0, 362, 3706, 344, 18, 338, 0] combine_rows_per_src=[8192, 3276, 7888, 8178, 8190, 8024, 8192, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=146628608 dispatch_meta_bytes=210040 combine_payload_bytes=146628608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=146838648 ingress_bytes=171794432 link_bytes=171794432 token_bytes=293467256 weight_movement_bytes=25165824 total_nvlink_bytes=318633080 weight_to_token_ratio=0.085753 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.307157130e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=194641920 dispatch_meta_bytes=211552 combine_payload_bytes=194641920 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=194853472 ingress_bytes=345636864 link_bytes=345636864 token_bytes=389495392 weight_movement_bytes=150994944 total_nvlink_bytes=540490336 weight_to_token_ratio=0.387668 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=7040 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.629897174e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=182394880 dispatch_meta_bytes=210752 combine_payload_bytes=182394880 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=182605632 ingress_bytes=232726528 link_bytes=232726528 token_bytes=365000512 weight_movement_bytes=50331648 total_nvlink_bytes=415332160 weight_to_token_ratio=0.137895 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.770779977e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=199778304 dispatch_meta_bytes=229328 combine_payload_bytes=199778304 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200007632 ingress_bytes=224944128 link_bytes=224944128 token_bytes=399785936 weight_movement_bytes=25165824 total_nvlink_bytes=424951760 weight_to_token_ratio=0.062948 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.711564906e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=177184768 dispatch_meta_bytes=206528 combine_payload_bytes=177184768 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=177391296 ingress_bytes=252682240 link_bytes=252682240 token_bytes=354576064 weight_movement_bytes=75497472 total_nvlink_bytes=430073536 weight_to_token_ratio=0.212923 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.922619888e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=213327872 dispatch_meta_bytes=229976 combine_payload_bytes=213327872 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=213557848 ingress_bytes=263659520 link_bytes=263659520 token_bytes=426885720 weight_movement_bytes=50331648 total_nvlink_bytes=477217368 weight_to_token_ratio=0.117904 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.006144305e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49816 payload_rows_remote=41740 dup_rows=15720 dup_rows_remote=7834 dedup_ratio=0.239868 router_tokens_per_expert=[0, 7, 0, 124, 0, 44, 0, 0, 0, 825, 116, 0, 0, 4, 0, 0, 0, 153, 1, 1, 4096, 0, 3847, 0, 2, 69, 2, 0, 4090, 26, 261, 2, 0, 1, 0, 0, 0, 0, 0, 43, 4, 0, 0, 0, 0, 12, 0, 0, 0, 3283, 0, 4096, 0, 3678, 0, 0, 0, 0, 3943, 0, 4037, 0, 0, 1] payload_rows_per_dst=[278, 1652, 7786, 8192, 8192, 8192, 7448, 8076] dup_rows_per_dst=[72, 238, 218, 712, 88, 32, 6474, 7886] combine_rows_per_src=[278, 1652, 7786, 8192, 8192, 8192, 7448, 8076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49490 payload_rows_remote=41312 dup_rows=16046 dup_rows_remote=8500 dedup_ratio=0.244843 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 4039, 0, 1, 0, 0, 3, 0, 48, 2, 6, 4073, 221, 813, 0, 4080, 0, 0, 0, 4087, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4086, 0, 3897, 0, 0, 0, 8, 3024, 0, 4096, 0, 0, 259] payload_rows_per_dst=[8192, 8086, 2132, 8178, 8146, 8174, 16, 6566] dup_rows_per_dst=[6, 7796, 680, 7546, 0, 2, 0, 16] combine_rows_per_src=[8192, 8086, 2132, 8178, 8146, 8174, 16, 6566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50548 payload_rows_remote=42356 dup_rows=14988 dup_rows_remote=12282 dedup_ratio=0.228699 router_tokens_per_expert=[18, 129, 228, 4044, 3976, 0, 34, 272, 3542, 668, 10, 520, 4052, 0, 0, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 74, 26, 123, 1, 146, 1, 3, 0, 0, 0, 2221, 0, 25, 6, 0, 0, 0, 49, 0, 3022, 606, 1434, 0, 1328, 0, 0, 0, 0, 21, 28, 28, 1, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[6900, 1940, 8104, 8148, 6652, 4220, 8192, 6392] dup_rows_per_dst=[1126, 456, 0, 5558, 3896, 1246, 2706, 0] combine_rows_per_src=[6900, 1940, 8104, 8148, 6652, 4220, 8192, 6392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49374 payload_rows_remote=41182 dup_rows=16162 dup_rows_remote=7922 dedup_ratio=0.246613 router_tokens_per_expert=[0, 0, 0, 1, 125, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 189, 32, 0, 3669, 23, 0, 4096, 0, 0, 2, 207, 55, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 26, 21, 0, 3, 120, 0, 0, 0, 0, 27, 0, 0, 0, 544, 7, 4057, 0, 3949, 0, 0, 0, 0, 3413, 0, 4093, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8172, 524, 8192, 8188, 5890, 2030] dup_rows_per_dst=[252, 8240, 7342, 0, 100, 220, 8, 0] combine_rows_per_src=[8186, 8192, 8172, 524, 8192, 8188, 5890, 2030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49624 payload_rows_remote=41488 dup_rows=15912 dup_rows_remote=7448 dedup_ratio=0.242798 router_tokens_per_expert=[0, 41, 14, 0, 0, 0, 1, 13, 42, 16, 0, 33, 0, 0, 0, 0, 6, 39, 0, 2519, 2311, 0, 1679, 0, 3, 0, 0, 201, 4057, 0, 0, 0, 9, 0, 0, 0, 4, 0, 8, 1, 105, 0, 0, 0, 0, 0, 0, 0, 2207, 1312, 1590, 4084, 0, 3851, 15, 0, 0, 0, 3858, 0, 4055, 0, 1, 693] payload_rows_per_dst=[8190, 7792, 8136, 408, 4644, 8180, 4476, 7798] dup_rows_per_dst=[58, 92, 8464, 0, 22, 198, 5772, 1306] combine_rows_per_src=[8190, 7792, 8136, 408, 4644, 8180, 4476, 7798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=170967040 dispatch_meta_bytes=198296 combine_payload_bytes=170967040 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=171165336 ingress_bytes=246464512 link_bytes=246464512 token_bytes=342132376 weight_movement_bytes=75497472 total_nvlink_bytes=417629848 weight_to_token_ratio=0.220667 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=23296 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.875310162e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56194 payload_rows_remote=48002 dup_rows=9342 dup_rows_remote=1172 dedup_ratio=0.142548 router_tokens_per_expert=[0, 5, 0, 0, 4054, 0, 1415, 0, 4093, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3941, 4076, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 22, 0, 1675, 0, 1407, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 3692] payload_rows_per_dst=[2836, 8186, 8114, 7882, 8192, 5406, 8192, 7386] dup_rows_per_dst=[4, 74, 4, 0, 8170, 802, 8, 280] combine_rows_per_src=[2836, 8186, 8114, 7882, 8192, 5406, 8192, 7386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44800 payload_rows_remote=36608 dup_rows=20736 dup_rows_remote=12556 dedup_ratio=0.316406 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 4084, 0, 563, 0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 468, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 3, 4096, 0, 4096, 0, 0, 0, 0, 3988, 0, 4096, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 946, 3202, 8192, 7884, 0] dup_rows_per_dst=[8180, 6610, 5556, 0, 374, 0, 16, 0] combine_rows_per_src=[8192, 8192, 8192, 946, 3202, 8192, 7884, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=169213952 dispatch_meta_bytes=199248 combine_payload_bytes=169213952 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=169413200 ingress_bytes=219545600 link_bytes=219545600 token_bytes=338627152 weight_movement_bytes=50331648 total_nvlink_bytes=388958800 weight_to_token_ratio=0.148634 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.670488345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50772 payload_rows_remote=42660 dup_rows=14764 dup_rows_remote=14750 dedup_ratio=0.225281 router_tokens_per_expert=[0, 0, 1, 14, 0, 0, 2, 0, 210, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 320, 0, 2654, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 4, 4031, 0, 1, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 687, 0, 4055, 0, 3675, 90, 0, 0, 0, 4072, 0, 4093, 0, 0, 89] payload_rows_per_dst=[8192, 1644, 640, 8176, 8180, 8112, 7636, 8192] dup_rows_per_dst=[28, 0, 0, 5272, 8052, 14, 1268, 130] combine_rows_per_src=[8192, 1644, 640, 8176, 8180, 8112, 7636, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=173490176 dispatch_meta_bytes=218552 combine_payload_bytes=173490176 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=173708728 ingress_bytes=198656000 link_bytes=198656000 token_bytes=347198904 weight_movement_bytes=25165824 total_nvlink_bytes=372364728 weight_to_token_ratio=0.072482 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.511542626e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=168681472 dispatch_meta_bytes=196416 combine_payload_bytes=168681472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=168877888 ingress_bytes=193847296 link_bytes=193847296 token_bytes=337559360 weight_movement_bytes=25165824 total_nvlink_bytes=362725184 weight_to_token_ratio=0.074552 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.474953944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=169934848 dispatch_meta_bytes=195744 combine_payload_bytes=169934848 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=170130592 ingress_bytes=220266496 link_bytes=220266496 token_bytes=340065440 weight_movement_bytes=50331648 total_nvlink_bytes=390397088 weight_to_token_ratio=0.148006 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.675973531e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=196616192 dispatch_meta_bytes=196696 combine_payload_bytes=196616192 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=196812888 ingress_bytes=246947840 link_bytes=246947840 token_bytes=393429080 weight_movement_bytes=50331648 total_nvlink_bytes=443760728 weight_to_token_ratio=0.127931 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.878987730e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=149946368 dispatch_meta_bytes=196656 combine_payload_bytes=149946368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=150143024 ingress_bytes=175112192 link_bytes=175112192 token_bytes=300089392 weight_movement_bytes=25165824 total_nvlink_bytes=325255216 weight_to_token_ratio=0.083861 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=28288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.332401450e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=174735360 dispatch_meta_bytes=229640 combine_payload_bytes=174735360 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=174965000 ingress_bytes=225067008 link_bytes=225067008 token_bytes=349700360 weight_movement_bytes=50331648 total_nvlink_bytes=400032008 weight_to_token_ratio=0.143928 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.712499880e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61272 payload_rows_remote=53146 dup_rows=4264 dup_rows_remote=3140 dedup_ratio=0.065063 router_tokens_per_expert=[0, 0, 0, 288, 4064, 0, 4055, 0, 3739, 2, 0, 0, 97, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 2, 1, 0, 0, 4073, 34, 0, 3565, 2899, 31, 0, 0, 92, 81, 10, 0, 0, 27, 0, 1045, 0, 3782, 15, 557, 0, 0, 0, 0, 0, 62, 175, 0, 0, 0, 14, 40, 0, 0, 0, 4014] payload_rows_per_dst=[8128, 7432, 8132, 7200, 6082, 7982, 8190, 8126] dup_rows_per_dst=[558, 12, 4, 4, 298, 1834, 430, 1124] combine_rows_per_src=[8128, 7432, 8132, 7200, 6082, 7982, 8190, 8126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51674 payload_rows_remote=45232 dup_rows=13862 dup_rows_remote=13728 dedup_ratio=0.211517 router_tokens_per_expert=[0, 26, 0, 0, 4012, 0, 1420, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 37, 1, 0, 53, 0, 42, 0, 4093, 0, 42, 3151, 552, 855, 0, 294, 0, 1, 0, 0, 0, 0, 4, 89, 0, 0, 150, 3934, 0, 0, 0, 0, 0, 2628, 0, 2, 0, 1, 0, 0, 0, 42, 0, 4060] payload_rows_per_dst=[5432, 4346, 8098, 6442, 2926, 8118, 8190, 8122] dup_rows_per_dst=[898, 0, 6776, 134, 478, 236, 5256, 84] combine_rows_per_src=[5432, 4346, 8098, 6442, 2926, 8118, 8190, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47800 payload_rows_remote=39812 dup_rows=17736 dup_rows_remote=17470 dedup_ratio=0.270630 router_tokens_per_expert=[0, 2493, 0, 10, 2140, 0, 0, 0, 3621, 0, 137, 0, 3978, 9, 3, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 38, 1275, 0, 4096, 0, 0, 4065, 4096, 0, 0, 223, 0, 0, 0, 0, 0, 15, 202, 0, 0, 4047, 0, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 11, 0, 0, 3, 200] payload_rows_per_dst=[7712, 7988, 4294, 8192, 2804, 8188, 8192, 430] dup_rows_per_dst=[4536, 266, 0, 8398, 0, 4530, 0, 6] combine_rows_per_src=[7712, 7988, 4294, 8192, 2804, 8188, 8192, 430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50224 payload_rows_remote=42032 dup_rows=15312 dup_rows_remote=10644 dedup_ratio=0.233643 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 310, 0, 480, 18, 0, 0, 0, 0, 4, 0, 0, 129, 4041, 29, 43, 0, 1630, 0, 1, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8, 82, 6, 0, 26, 0, 1394, 0, 407, 4064, 50, 4051, 528, 0, 0, 0, 3282, 0, 4086, 0, 0, 4063] payload_rows_per_dst=[7518, 1616, 402, 8014, 8128, 8192, 8192, 8162] dup_rows_per_dst=[0, 0, 8, 3260, 12, 180, 4668, 7184] combine_rows_per_src=[7518, 1616, 402, 8014, 8128, 8192, 8192, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53832 payload_rows_remote=45730 dup_rows=11704 dup_rows_remote=11576 dedup_ratio=0.178589 router_tokens_per_expert=[0, 0, 0, 71, 3960, 0, 3622, 1, 4032, 0, 2432, 0, 1, 3, 0, 0, 0, 0, 0, 54, 0, 101, 0, 0, 0, 0, 0, 0, 4009, 15, 154, 2, 763, 0, 0, 0, 0, 3892, 0, 0, 6, 0, 0, 0, 115, 398, 3, 4075, 0, 0, 3, 0, 0, 0, 5, 953, 0, 0, 0, 0, 0, 0, 3, 4095] payload_rows_per_dst=[8192, 4872, 8102, 316, 7784, 8188, 8188, 8190] dup_rows_per_dst=[7260, 0, 128, 26, 1526, 1006, 1752, 6] combine_rows_per_src=[8192, 4872, 8102, 316, 7784, 8188, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=217686016 dispatch_meta_bytes=225144 combine_payload_bytes=217686016 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=217911160 ingress_bytes=242851840 link_bytes=242851840 token_bytes=435597176 weight_movement_bytes=25165824 total_nvlink_bytes=460763000 weight_to_token_ratio=0.057773 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.847821903e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49330 payload_rows_remote=41138 dup_rows=16206 dup_rows_remote=16040 dedup_ratio=0.247284 router_tokens_per_expert=[34, 0, 115, 0, 0, 0, 4093, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4014, 0, 3954, 0, 83, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 5, 322, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 3970, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6378, 8176, 8192, 8140, 8192, 2218, 230, 7804] dup_rows_per_dst=[68, 7930, 234, 7796, 166, 12, 0, 0] combine_rows_per_src=[6378, 8176, 8192, 8140, 8192, 2218, 230, 7804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=185270272 dispatch_meta_bytes=235840 combine_payload_bytes=185270272 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=185506112 ingress_bytes=235601920 link_bytes=235601920 token_bytes=370776384 weight_movement_bytes=50331648 total_nvlink_bytes=421108032 weight_to_token_ratio=0.135747 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792658388e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48636 payload_rows_remote=40486 dup_rows=16900 dup_rows_remote=13374 dedup_ratio=0.257874 router_tokens_per_expert=[0, 3645, 0, 0, 4096, 4, 61, 0, 857, 0, 1067, 0, 3409, 0, 0, 0, 0, 0, 0, 1868, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 118, 4080, 1919, 0, 0, 896, 21, 0, 0, 0, 0, 0, 0, 0, 799, 0, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096] payload_rows_per_dst=[8150, 7704, 6708, 8192, 4300, 4344, 1046, 8192] dup_rows_per_dst=[3526, 1248, 2694, 7356, 1372, 700, 0, 4] combine_rows_per_src=[8150, 7704, 6708, 8192, 4300, 4344, 1046, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51398 payload_rows_remote=47680 dup_rows=14138 dup_rows_remote=13420 dedup_ratio=0.215729 router_tokens_per_expert=[0, 4, 0, 4, 3786, 0, 0, 0, 4092, 0, 167, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 39, 0, 8, 1, 0, 25, 6, 4071, 0, 0, 3981, 4095, 1434, 0, 6, 0, 4060, 0, 0, 0, 3, 0, 0, 0, 508, 0, 2553, 0, 0, 0, 0, 0, 1, 1, 0, 1, 8, 0, 0, 0, 120, 0, 3753] payload_rows_per_dst=[12, 8192, 7650, 7966, 8190, 3718, 8146, 7524] dup_rows_per_dst=[4, 408, 16, 60, 11000, 718, 0, 1932] combine_rows_per_src=[12, 8192, 7650, 7966, 8190, 3718, 8146, 7524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=163069952 dispatch_meta_bytes=229128 combine_payload_bytes=163069952 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=163299080 ingress_bytes=238567424 link_bytes=238567424 token_bytes=326369032 weight_movement_bytes=75497472 total_nvlink_bytes=401866504 weight_to_token_ratio=0.231325 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=18048 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.815222447e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=172163072 dispatch_meta_bytes=210704 combine_payload_bytes=172163072 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=172373776 ingress_bytes=272826368 link_bytes=272826368 token_bytes=344536848 weight_movement_bytes=100663296 total_nvlink_bytes=445200144 weight_to_token_ratio=0.292170 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=12544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.075893427e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=187310080 dispatch_meta_bytes=229224 combine_payload_bytes=187310080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=187539304 ingress_bytes=212475904 link_bytes=212475904 token_bytes=374849384 weight_movement_bytes=25165824 total_nvlink_bytes=400015208 weight_to_token_ratio=0.067136 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.616696127e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=168501248 dispatch_meta_bytes=228712 combine_payload_bytes=168501248 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=168729960 ingress_bytes=218832896 link_bytes=218832896 token_bytes=337231208 weight_movement_bytes=50331648 total_nvlink_bytes=387562856 weight_to_token_ratio=0.149250 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=29312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.665065491e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=165830656 dispatch_meta_bytes=215440 combine_payload_bytes=165830656 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=166046096 ingress_bytes=216162304 link_bytes=216162304 token_bytes=331876752 weight_movement_bytes=50331648 total_nvlink_bytes=382208400 weight_to_token_ratio=0.151658 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=13312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.644745372e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=195297280 dispatch_meta_bytes=244400 combine_payload_bytes=195297280 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=195541680 ingress_bytes=245628928 link_bytes=245628928 token_bytes=390838960 weight_movement_bytes=50331648 total_nvlink_bytes=441170608 weight_to_token_ratio=0.128778 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.868952334e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=37040 payload_rows_remote=28848 dup_rows=28496 dup_rows_remote=15078 dedup_ratio=0.434814 router_tokens_per_expert=[787, 0, 4071, 0, 0, 285, 3395, 0, 0, 0, 1, 0, 0, 0, 0, 0, 24, 3959, 0, 0, 0, 0, 1, 4044, 0, 0, 4, 1, 0, 3335, 0, 0, 0, 0, 383, 124, 94, 0, 0, 0, 0, 0, 0, 807, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 3471, 3995, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[2098, 2, 7968, 8, 2402, 8186, 8184, 8192] dup_rows_per_dst=[46, 0, 0, 2, 414, 7760, 6856, 13418] combine_rows_per_src=[2098, 2, 7968, 8, 2402, 8186, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51086 payload_rows_remote=42898 dup_rows=14450 dup_rows_remote=13972 dedup_ratio=0.220490 router_tokens_per_expert=[4, 0, 4090, 46, 0, 0, 1080, 0, 0, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 3972, 5, 30, 49, 0, 8, 4078, 6, 157, 3, 0, 0, 0, 0, 3, 0, 0, 0, 545, 0, 4062, 0, 3132, 0, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[2260, 8192, 7954, 8188, 8132, 8178, 8180, 2] dup_rows_per_dst=[0, 8158, 0, 478, 4, 5810, 0, 0] combine_rows_per_src=[2260, 8192, 7954, 8188, 8132, 8178, 8180, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55370 payload_rows_remote=53566 dup_rows=10166 dup_rows_remote=10166 dedup_ratio=0.155121 router_tokens_per_expert=[2363, 0, 4096, 0, 190, 418, 4028, 0, 0, 0, 16, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4083, 0, 0, 2, 0, 0, 928, 0, 4096, 0, 0, 0, 62, 0, 0, 22, 0, 1, 62, 0, 4013, 0, 2511, 0, 0, 0, 2, 0, 0, 0, 11, 0, 0, 3299, 877, 0, 149, 0, 34, 1494, 0] payload_rows_per_dst=[8182, 1804, 8170, 8192, 8156, 5148, 8192, 7526] dup_rows_per_dst=[5816, 0, 0, 1860, 38, 0, 26, 2426] combine_rows_per_src=[8182, 1804, 8170, 8192, 8156, 5148, 8192, 7526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=118161408 dispatch_meta_bytes=175704 combine_payload_bytes=118161408 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=118337112 ingress_bytes=168493056 link_bytes=168493056 token_bytes=236498520 weight_movement_bytes=50331648 total_nvlink_bytes=286830168 weight_to_token_ratio=0.212820 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.282037473e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49024 payload_rows_remote=41088 dup_rows=16512 dup_rows_remote=16502 dedup_ratio=0.251953 router_tokens_per_expert=[0, 1392, 0, 0, 2362, 4, 2698, 50, 3986, 0, 2739, 0, 679, 0, 0, 0, 0, 4, 0, 0, 9, 33, 0, 0, 1294, 3, 0, 75, 3946, 0, 0, 312, 14, 0, 4, 0, 0, 1152, 28, 9, 0, 843, 0, 0, 0, 0, 4040, 2964, 4, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 5, 0, 4096] payload_rows_per_dst=[6218, 8184, 4812, 3210, 2382, 8090, 7936, 8192] dup_rows_per_dst=[2070, 6624, 4, 158, 32, 1676, 10, 5938] combine_rows_per_src=[6218, 8184, 4812, 3210, 2382, 8090, 7936, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45584 payload_rows_remote=37392 dup_rows=19952 dup_rows_remote=11820 dedup_ratio=0.304443 router_tokens_per_expert=[3032, 0, 3907, 0, 512, 0, 3766, 284, 4, 0, 4, 0, 4, 0, 0, 0, 0, 4056, 0, 2, 9, 0, 0, 4095, 4, 136, 171, 0, 0, 3087, 2, 61, 0, 1525, 34, 0, 0, 0, 0, 0, 13, 2668, 0, 4080, 0, 13, 0, 591, 0, 3, 0, 0, 0, 0, 0, 0, 130, 319, 2, 0, 88, 1, 165, 0] payload_rows_per_dst=[8190, 652, 8192, 6638, 8160, 5336, 7820, 596] dup_rows_per_dst=[6998, 10, 8132, 284, 3118, 1234, 0, 176] combine_rows_per_src=[8190, 652, 8192, 6638, 8160, 5336, 7820, 596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53100 payload_rows_remote=44908 dup_rows=12436 dup_rows_remote=8336 dedup_ratio=0.189758 router_tokens_per_expert=[0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 2741, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2050, 0, 4096, 0, 0, 0, 0, 3035, 2, 0, 0, 0, 0, 0, 7, 2138, 1694, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6766, 7984, 8192, 8192, 6070, 3428, 4276] dup_rows_per_dst=[1152, 4088, 3012, 0, 4100, 18, 66, 0] combine_rows_per_src=[8192, 6766, 7984, 8192, 8192, 6070, 3428, 4276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=175710208 dispatch_meta_bytes=227480 combine_payload_bytes=175710208 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=175937688 ingress_bytes=226041856 link_bytes=226041856 token_bytes=351647896 weight_movement_bytes=50331648 total_nvlink_bytes=401979544 weight_to_token_ratio=0.143131 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.719917347e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=219406336 dispatch_meta_bytes=254928 combine_payload_bytes=219406336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=219661264 ingress_bytes=244572160 link_bytes=244572160 token_bytes=439067600 weight_movement_bytes=25165824 total_nvlink_bytes=464233424 weight_to_token_ratio=0.057317 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=19072 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.860911550e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61488 payload_rows_remote=53306 dup_rows=4048 dup_rows_remote=1046 dedup_ratio=0.061768 router_tokens_per_expert=[3492, 0, 3414, 1, 0, 0, 4096, 0, 0, 0, 0, 3802, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4065, 0, 266, 0, 0, 25, 3603, 0, 0, 0, 1, 0, 0, 0, 0, 276, 5, 0, 6, 0, 4095, 0, 1520, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4060, 3, 1, 27, 0, 0, 0] payload_rows_per_dst=[6984, 8184, 8192, 7502, 8192, 8182, 6844, 7408] dup_rows_per_dst=[2, 188, 2, 286, 562, 3002, 0, 6] combine_rows_per_src=[6984, 8184, 8192, 7502, 8192, 8182, 6844, 7408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50358 payload_rows_remote=42170 dup_rows=15178 dup_rows_remote=9928 dedup_ratio=0.231598 router_tokens_per_expert=[4094, 0, 4060, 0, 0, 0, 2625, 0, 0, 0, 0, 3934, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 60, 0, 0, 63, 0, 45, 0, 823, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4096, 37, 620, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[8188, 8192, 8192, 240, 8192, 1040, 8122, 8192] dup_rows_per_dst=[5250, 7872, 0, 96, 1926, 28, 2, 4] combine_rows_per_src=[8188, 8192, 8192, 240, 8192, 1040, 8122, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=168296448 dispatch_meta_bytes=230360 combine_payload_bytes=168296448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=168526808 ingress_bytes=193462272 link_bytes=193462272 token_bytes=336823256 weight_movement_bytes=25165824 total_nvlink_bytes=361989080 weight_to_token_ratio=0.074715 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28928 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.472024357e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=153157632 dispatch_meta_bytes=196848 combine_payload_bytes=153157632 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=153354480 ingress_bytes=203489280 link_bytes=203489280 token_bytes=306512112 weight_movement_bytes=50331648 total_nvlink_bytes=356843760 weight_to_token_ratio=0.164208 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.548318302e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=183943168 dispatch_meta_bytes=212976 combine_payload_bytes=183943168 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=184156144 ingress_bytes=259440640 link_bytes=259440640 token_bytes=368099312 weight_movement_bytes=75497472 total_nvlink_bytes=443596784 weight_to_token_ratio=0.205101 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.974043503e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=218341376 dispatch_meta_bytes=217408 combine_payload_bytes=218341376 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=218558784 ingress_bytes=293838848 link_bytes=293838848 token_bytes=436900160 weight_movement_bytes=75497472 total_nvlink_bytes=512397632 weight_to_token_ratio=0.172803 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=13056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.235774121e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=172728320 dispatch_meta_bytes=208392 combine_payload_bytes=172728320 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=172936712 ingress_bytes=273391616 link_bytes=273391616 token_bytes=345665032 weight_movement_bytes=100663296 total_nvlink_bytes=446328328 weight_to_token_ratio=0.291216 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=16256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.080194311e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47742 payload_rows_remote=39748 dup_rows=17794 dup_rows_remote=10410 dedup_ratio=0.271515 router_tokens_per_expert=[0, 0, 0, 0, 4096, 270, 0, 1, 15, 22, 0, 42, 0, 0, 127, 0, 3129, 0, 2216, 0, 0, 12, 0, 1, 0, 96, 5, 0, 4, 773, 0, 0, 3727, 0, 8, 4082, 2, 0, 0, 0, 43, 0, 0, 2311, 0, 4, 0, 0, 3960, 505, 0, 0, 0, 0, 3, 4090, 0, 0, 0, 0, 0, 0, 336, 2888] payload_rows_per_dst=[8192, 254, 6856, 6690, 1520, 8050, 8186, 7994] dup_rows_per_dst=[542, 158, 3860, 1728, 2, 4120, 0, 7384] combine_rows_per_src=[8192, 254, 6856, 6690, 1520, 8050, 8186, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60530 payload_rows_remote=52354 dup_rows=5006 dup_rows_remote=5006 dedup_ratio=0.076385 router_tokens_per_expert=[0, 0, 0, 26, 2276, 0, 0, 0, 0, 0, 0, 0, 89, 0, 1, 0, 35, 0, 4081, 0, 0, 0, 3463, 38, 0, 0, 0, 0, 0, 0, 0, 2, 3870, 174, 0, 4086, 0, 17, 0, 0, 4095, 0, 0, 0, 0, 12, 0, 0, 3974, 4096, 0, 2, 0, 0, 0, 158, 0, 2269, 0, 0, 0, 0, 0, 4] payload_rows_per_dst=[4622, 8168, 7002, 8176, 8072, 8192, 8192, 8106] dup_rows_per_dst=[52, 174, 0, 0, 50, 22, 320, 4388] combine_rows_per_src=[4622, 8168, 7002, 8176, 8072, 8192, 8192, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49062 payload_rows_remote=40880 dup_rows=16474 dup_rows_remote=13530 dedup_ratio=0.251373 router_tokens_per_expert=[0, 0, 0, 2, 3962, 0, 2, 0, 0, 0, 1617, 0, 0, 2, 19, 0, 4083, 0, 3925, 0, 0, 0, 805, 0, 5, 0, 0, 0, 0, 90, 36, 0, 4088, 785, 0, 4092, 0, 0, 1, 8, 3414, 0, 0, 0, 0, 0, 0, 8, 110, 1667, 0, 0, 44, 1, 0, 3986, 0, 0, 0, 0, 0, 8, 0, 8] payload_rows_per_dst=[8170, 8182, 1610, 8188, 8192, 6844, 7656, 220] dup_rows_per_dst=[7928, 2944, 0, 258, 1978, 0, 3334, 32] combine_rows_per_src=[8170, 8182, 1610, 8188, 8192, 6844, 7656, 220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=162807808 dispatch_meta_bytes=200632 combine_payload_bytes=162807808 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=163008440 ingress_bytes=213139456 link_bytes=213139456 token_bytes=325816248 weight_movement_bytes=50331648 total_nvlink_bytes=376147896 weight_to_token_ratio=0.154479 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.621744991e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49532 payload_rows_remote=41456 dup_rows=16004 dup_rows_remote=13000 dedup_ratio=0.244202 router_tokens_per_expert=[2379, 0, 3830, 0, 0, 159, 3880, 0, 0, 0, 0, 3081, 7, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 4054, 15, 0, 265, 1, 0, 4069, 0, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 157, 10, 60, 3941, 0, 0, 360, 60, 0, 11, 0, 0, 0, 0, 201, 24, 0, 3398, 0, 0, 0, 0, 111, 1] payload_rows_per_dst=[5052, 6176, 306, 6824, 7882, 8192, 8076, 7024] dup_rows_per_dst=[24, 0, 2, 9002, 0, 1210, 3004, 2762] combine_rows_per_src=[5052, 6176, 306, 6824, 7882, 8192, 8076, 7024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50124 payload_rows_remote=50118 dup_rows=15412 dup_rows_remote=15412 dedup_ratio=0.235168 router_tokens_per_expert=[0, 2, 0, 0, 7, 0, 11, 35, 0, 0, 269, 0, 3794, 0, 0, 0, 2160, 0, 3983, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 22, 0, 2, 3655, 0, 0, 3962, 0, 0, 0, 2, 0, 0, 0, 3039, 43, 0, 0, 134, 4096, 3810, 2, 1, 0, 0, 0, 3719, 13, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4364, 8192, 6, 7972, 7314, 6110, 7974, 8192] dup_rows_per_dst=[66, 7900, 0, 4, 0, 322, 7090, 30] combine_rows_per_src=[4364, 8192, 6, 7972, 7314, 6110, 7974, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=214441984 dispatch_meta_bytes=229440 combine_payload_bytes=214441984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=214671424 ingress_bytes=239607808 link_bytes=239607808 token_bytes=429113408 weight_movement_bytes=25165824 total_nvlink_bytes=454279232 weight_to_token_ratio=0.058646 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.823138567e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54310 payload_rows_remote=52204 dup_rows=11226 dup_rows_remote=11032 dedup_ratio=0.171295 router_tokens_per_expert=[0, 0, 0, 0, 4096, 3416, 0, 0, 2, 0, 0, 0, 0, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 0, 0, 1, 0, 0, 0, 0, 0, 170, 8, 846, 0, 0, 5, 121, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4046, 0, 3718, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6832, 8192, 4958, 8106, 2106, 8192, 7732, 8192] dup_rows_per_dst=[0, 3608, 0, 6076, 194, 0, 1348, 0] combine_rows_per_src=[6832, 8192, 4958, 8106, 2106, 8192, 7732, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=167444480 dispatch_meta_bytes=217640 combine_payload_bytes=167444480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=167662120 ingress_bytes=192610304 link_bytes=192610304 token_bytes=335106600 weight_movement_bytes=25165824 total_nvlink_bytes=360272424 weight_to_token_ratio=0.075098 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.465541865e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52722 payload_rows_remote=50618 dup_rows=12814 dup_rows_remote=12796 dedup_ratio=0.195526 router_tokens_per_expert=[0, 0, 3, 0, 4088, 0, 0, 0, 0, 0, 1047, 0, 3506, 0, 0, 0, 4, 0, 3488, 0, 0, 0, 3884, 0, 0, 0, 0, 6, 1, 0, 1, 0, 4096, 0, 1, 4034, 0, 0, 1, 0, 9, 0, 0, 791, 0, 261, 0, 0, 1893, 2399, 0, 0, 0, 0, 0, 2668, 0, 530, 0, 0, 0, 56, 0, 1] payload_rows_per_dst=[8182, 7016, 8150, 8080, 8192, 2104, 6646, 4352] dup_rows_per_dst=[0, 2090, 6602, 4, 4, 18, 3488, 608] combine_rows_per_src=[8182, 7016, 8150, 8080, 8192, 2104, 6646, 4352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55202 payload_rows_remote=47010 dup_rows=10334 dup_rows_remote=10334 dedup_ratio=0.157684 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 304, 0, 125, 0, 0, 0, 4095, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 4096, 0, 0, 0, 0, 3218, 126, 0, 0, 0, 0, 0, 41, 3320, 3905, 0, 0, 0, 0, 0, 588, 0, 714, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7558, 8192, 8192, 6688, 252, 7936] dup_rows_per_dst=[0, 770, 0, 0, 9350, 82, 0, 132] combine_rows_per_src=[8192, 8192, 7558, 8192, 8192, 6688, 252, 7936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=169803776 dispatch_meta_bytes=217824 combine_payload_bytes=169803776 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=170021600 ingress_bytes=220135424 link_bytes=220135424 token_bytes=339825376 weight_movement_bytes=50331648 total_nvlink_bytes=390157024 weight_to_token_ratio=0.148110 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.674976224e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=205283328 dispatch_meta_bytes=262120 combine_payload_bytes=205283328 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=205545448 ingress_bytes=280780800 link_bytes=280780800 token_bytes=410828776 weight_movement_bytes=75497472 total_nvlink_bytes=486326248 weight_to_token_ratio=0.183769 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.136417464e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=213827584 dispatch_meta_bytes=252944 combine_payload_bytes=213827584 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=214080528 ingress_bytes=264159232 link_bytes=264159232 token_bytes=427908112 weight_movement_bytes=50331648 total_nvlink_bytes=478239760 weight_to_token_ratio=0.117623 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.009946536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=207331328 dispatch_meta_bytes=253656 combine_payload_bytes=207331328 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=207584984 ingress_bytes=232497152 link_bytes=232497152 token_bytes=414916312 weight_movement_bytes=25165824 total_nvlink_bytes=440082136 weight_to_token_ratio=0.060653 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.769034691e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=192552960 dispatch_meta_bytes=229376 combine_payload_bytes=192552960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=192782336 ingress_bytes=217718784 link_bytes=217718784 token_bytes=385335296 weight_movement_bytes=25165824 total_nvlink_bytes=410501120 weight_to_token_ratio=0.065309 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.656588386e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52490 payload_rows_remote=44304 dup_rows=13046 dup_rows_remote=10768 dedup_ratio=0.199066 router_tokens_per_expert=[0, 0, 0, 2, 3199, 3740, 0, 0, 0, 386, 0, 0, 0, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 9, 0, 0, 0, 0, 2230, 19, 4091, 0, 11, 0, 0, 0, 2141, 0, 0, 0, 3500, 40, 0, 60, 0, 3767, 0, 1667, 0, 0, 582, 99, 0, 0, 11, 0, 0, 1935, 87, 0] payload_rows_per_dst=[5904, 2688, 8192, 7572, 6670, 7188, 6090, 8186] dup_rows_per_dst=[0, 238, 0, 18, 3556, 4294, 2662, 2278] combine_rows_per_src=[5904, 2688, 8192, 7572, 6670, 7188, 6090, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56886 payload_rows_remote=48908 dup_rows=8650 dup_rows_remote=900 dedup_ratio=0.131989 router_tokens_per_expert=[0, 0, 0, 12, 4093, 4050, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 2, 0, 0, 0, 0, 0, 170, 2, 0, 30, 0, 0, 0, 1, 0, 0, 49, 2, 1, 77, 7, 4095, 0, 1817, 0, 0, 2300, 4062, 1, 0, 0, 0, 3914, 0, 3917, 27, 0, 248, 3671, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8124, 8130, 340, 7978, 8192, 8090, 7846, 8186] dup_rows_per_dst=[0, 434, 4, 7750, 270, 146, 46, 0] combine_rows_per_src=[8124, 8130, 340, 7978, 8192, 8090, 7846, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55784 payload_rows_remote=47976 dup_rows=9752 dup_rows_remote=9752 dedup_ratio=0.148804 router_tokens_per_expert=[0, 1, 0, 36, 4028, 3858, 0, 5, 0, 0, 59, 0, 0, 0, 0, 0, 0, 1104, 0, 0, 3, 15, 0, 3992, 0, 0, 0, 3, 50, 0, 0, 25, 3826, 0, 5, 143, 1, 1, 221, 0, 3978, 0, 0, 160, 3845, 0, 144, 0, 0, 727, 0, 2694, 0, 0, 67, 3722, 0, 0, 19, 0, 0, 0, 33, 3] payload_rows_per_dst=[7788, 7808, 8186, 6096, 7696, 8180, 1898, 8132] dup_rows_per_dst=[12, 0, 3624, 902, 698, 4482, 0, 34] combine_rows_per_src=[7788, 7808, 8186, 6096, 7696, 8180, 1898, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=181469184 dispatch_meta_bytes=220288 combine_payload_bytes=181469184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=181689472 ingress_bytes=206635008 link_bytes=206635008 token_bytes=363158656 weight_movement_bytes=25165824 total_nvlink_bytes=388324480 weight_to_token_ratio=0.069297 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.572253657e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=200327168 dispatch_meta_bytes=199232 combine_payload_bytes=200327168 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=200526400 ingress_bytes=250658816 link_bytes=250658816 token_bytes=400853568 weight_movement_bytes=50331648 total_nvlink_bytes=451185216 weight_to_token_ratio=0.125561 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.907223970e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=39256 dup_rows=21142 dup_rows_remote=17376 dedup_ratio=0.322601 router_tokens_per_expert=[0, 0, 0, 0, 2907, 0, 0, 34, 0, 0, 61, 0, 227, 33, 0, 0, 4056, 6, 2382, 0, 4, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 24, 3538, 0, 0, 3914, 1, 0, 0, 0, 3920, 0, 0, 859, 2, 302, 0, 15, 4096, 4049, 195, 0, 0, 0, 0, 1758, 67, 0, 0, 135, 95, 0, 0, 0] payload_rows_per_dst=[5882, 608, 8126, 8004, 378, 8066, 5138, 8192] dup_rows_per_dst=[0, 34, 4770, 48, 0, 8830, 3766, 3694] combine_rows_per_src=[5882, 608, 8126, 8004, 378, 8066, 5138, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57606 payload_rows_remote=49414 dup_rows=7930 dup_rows_remote=4244 dedup_ratio=0.121002 router_tokens_per_expert=[39, 0, 0, 0, 4040, 4096, 0, 312, 0, 0, 90, 0, 0, 1119, 0, 0, 0, 4096, 0, 0, 0, 0, 385, 1458, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4011, 2, 154, 0, 14, 0, 4063, 0, 0, 0, 3926, 0, 0, 0, 0, 54, 0, 573, 1, 0, 30, 3714, 0, 0, 0, 0, 5, 569, 0, 16] payload_rows_per_dst=[8192, 8032, 8192, 1256, 8178, 8126, 7488, 8142] dup_rows_per_dst=[702, 2238, 3686, 0, 184, 0, 2, 1118] combine_rows_per_src=[8192, 8032, 8192, 1256, 8178, 8126, 7488, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55828 payload_rows_remote=47640 dup_rows=9708 dup_rows_remote=9604 dedup_ratio=0.148132 router_tokens_per_expert=[0, 0, 75, 4005, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 63, 4096, 4017, 0, 0, 3, 0, 0, 0, 0, 0, 9, 0, 2760, 0, 62, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1298, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[1274, 8192, 8192, 5538, 8188, 8192, 8192, 8060] dup_rows_per_dst=[0, 0, 8166, 0, 104, 14, 0, 1424] combine_rows_per_src=[1274, 8192, 8192, 5538, 8188, 8192, 8192, 8060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=196509696 dispatch_meta_bytes=230912 combine_payload_bytes=196509696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=196740608 ingress_bytes=221675520 link_bytes=221675520 token_bytes=393250304 weight_movement_bytes=25165824 total_nvlink_bytes=418416128 weight_to_token_ratio=0.063994 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=24192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.686694575e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54688 payload_rows_remote=46898 dup_rows=10848 dup_rows_remote=10848 dedup_ratio=0.165527 router_tokens_per_expert=[0, 1, 0, 0, 4073, 4092, 0, 189, 6, 0, 0, 0, 0, 1467, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 675, 2273, 0, 0, 45, 0, 0, 3891, 0, 0, 2, 4070, 2, 0, 0, 0, 4071, 0, 3754, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 74, 0, 4] payload_rows_per_dst=[8192, 8192, 310, 8184, 5910, 7790, 7918, 8192] dup_rows_per_dst=[372, 2894, 0, 12, 76, 0, 7384, 110] combine_rows_per_src=[8192, 8192, 310, 8184, 5910, 7790, 7918, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=41664 payload_rows_remote=33472 dup_rows=23872 dup_rows_remote=20450 dedup_ratio=0.364258 router_tokens_per_expert=[0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2308, 0, 0, 0, 1880, 0, 0, 0, 4, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 20, 188, 0, 1941, 0, 0, 0, 4093, 0, 4091, 0, 0, 0, 4093, 0, 0, 0, 0, 1476, 0, 0, 0, 0, 0, 500, 0, 0, 0, 0, 2, 72, 0, 0] payload_rows_per_dst=[8192, 8186, 8190, 2992, 8192, 1000, 0, 4912] dup_rows_per_dst=[3422, 4616, 11578, 0, 4252, 0, 0, 4] combine_rows_per_src=[8192, 8186, 8190, 2992, 8192, 1000, 0, 4912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=160792576 dispatch_meta_bytes=226528 combine_payload_bytes=160792576 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=161019104 ingress_bytes=261455872 link_bytes=261455872 token_bytes=321811680 weight_movement_bytes=100663296 total_nvlink_bytes=422474976 weight_to_token_ratio=0.312802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.989377090e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=202399744 dispatch_meta_bytes=214632 combine_payload_bytes=202399744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202614376 ingress_bytes=227565568 link_bytes=227565568 token_bytes=405014120 weight_movement_bytes=25165824 total_nvlink_bytes=430179944 weight_to_token_ratio=0.062136 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.731511035e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=195133440 dispatch_meta_bytes=228976 combine_payload_bytes=195133440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=195362416 ingress_bytes=220299264 link_bytes=220299264 token_bytes=390495856 weight_movement_bytes=25165824 total_nvlink_bytes=415661680 weight_to_token_ratio=0.064446 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27520 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.676222857e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=192094208 dispatch_meta_bytes=230984 combine_payload_bytes=192094208 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=192325192 ingress_bytes=267591680 link_bytes=267591680 token_bytes=384419400 weight_movement_bytes=75497472 total_nvlink_bytes=459916872 weight_to_token_ratio=0.196394 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.036063500e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56646 payload_rows_remote=48458 dup_rows=8890 dup_rows_remote=8116 dedup_ratio=0.135651 router_tokens_per_expert=[2087, 65, 0, 3988, 0, 4092, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 107, 0, 4039, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 1765, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 91, 0, 0, 0, 0, 0, 8, 0, 3869, 0, 21, 0, 0, 3967, 0, 0, 2, 0, 452, 0, 34, 0, 7, 0, 0] payload_rows_per_dst=[4278, 8192, 8192, 3530, 8192, 8094, 7980, 8188] dup_rows_per_dst=[26, 2, 166, 0, 184, 7738, 0, 774] combine_rows_per_src=[4278, 8192, 8192, 3530, 8192, 8094, 7980, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=137101312 dispatch_meta_bytes=215688 combine_payload_bytes=137101312 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=137317000 ingress_bytes=187432960 link_bytes=187432960 token_bytes=274418312 weight_movement_bytes=50331648 total_nvlink_bytes=324749960 weight_to_token_ratio=0.183412 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.426148259e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64018 payload_rows_remote=55836 dup_rows=1518 dup_rows_remote=1122 dedup_ratio=0.023163 router_tokens_per_expert=[0, 6, 5, 4092, 0, 4087, 3, 0, 0, 0, 0, 3982, 0, 0, 0, 0, 217, 0, 3412, 976, 0, 1, 0, 0, 0, 0, 0, 0, 198, 0, 4091, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 1, 32, 0, 3994, 0, 0, 0, 0, 3927, 0, 70, 0, 0, 3664, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[8192, 7964, 8192, 8182, 8182, 8052, 7924, 7330] dup_rows_per_dst=[20, 0, 1020, 396, 0, 2, 70, 10] combine_rows_per_src=[8192, 7964, 8192, 8182, 8182, 8052, 7924, 7330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49064 payload_rows_remote=43228 dup_rows=16472 dup_rows_remote=16472 dedup_ratio=0.251343 router_tokens_per_expert=[0, 39, 0, 4077, 0, 4007, 0, 5, 0, 0, 0, 2918, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 0, 309, 0, 0, 0, 0, 0, 3891, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 1198, 0, 0, 0, 0, 3783, 0, 1795, 80, 0, 4091, 0, 0, 0, 0, 2315, 0] payload_rows_per_dst=[8192, 5836, 8178, 1962, 6110, 2412, 8192, 8182] dup_rows_per_dst=[8690, 0, 2, 0, 0, 26, 3124, 4630] combine_rows_per_src=[8192, 5836, 8178, 1962, 6110, 2412, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6095757192 prefetch_bytes=352321536 reduce_bytes=402653184 weight_movement_bytes=754974720 egress_bytes=3049684872 ingress_bytes=3801047040 link_bytes=3801047040 total_nvlink_bytes=6850731912 weight_to_token_ratio=0.123852 prefetch_share_of_ingress=0.092691 dedup_ratio=0.179281 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=16 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.037684 gemm_inner_pad_upper_bound=19558 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.807598507e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=198483968 dispatch_meta_bytes=226296 combine_payload_bytes=198483968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198710264 ingress_bytes=223649792 link_bytes=223649792 token_bytes=397194232 weight_movement_bytes=25165824 total_nvlink_bytes=422360056 weight_to_token_ratio=0.063359 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=15488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.701716504e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228704256 dispatch_meta_bytes=227832 combine_payload_bytes=228704256 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=228932088 ingress_bytes=279035904 link_bytes=279035904 token_bytes=457636344 weight_movement_bytes=50331648 total_nvlink_bytes=507967992 weight_to_token_ratio=0.109982 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.123140821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=177061888 dispatch_meta_bytes=238800 combine_payload_bytes=177061888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=177300688 ingress_bytes=202227712 link_bytes=202227712 token_bytes=354362576 weight_movement_bytes=25165824 total_nvlink_bytes=379528400 weight_to_token_ratio=0.071017 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.538719227e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53992 payload_rows_remote=45802 dup_rows=11544 dup_rows_remote=11334 dedup_ratio=0.176147 router_tokens_per_expert=[0, 0, 2, 4018, 81, 4072, 0, 0, 0, 0, 0, 4060, 0, 0, 23, 0, 0, 0, 4031, 0, 43, 0, 126, 0, 0, 0, 0, 0, 1606, 0, 3848, 0, 1215, 0, 0, 0, 0, 2, 2679, 0, 0, 0, 0, 3, 0, 0, 0, 956, 1, 0, 5, 0, 3485, 29, 12, 47, 0, 963, 0, 0, 0, 1, 1460, 0] payload_rows_per_dst=[8190, 8120, 8190, 7696, 8192, 1918, 7076, 4610] dup_rows_per_dst=[12, 46, 210, 3212, 7744, 0, 82, 238] combine_rows_per_src=[8190, 8120, 8190, 7696, 8192, 1918, 7076, 4610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56624 payload_rows_remote=48464 dup_rows=8912 dup_rows_remote=1950 dedup_ratio=0.135986 router_tokens_per_expert=[0, 0, 0, 1, 3931, 3899, 0, 2, 0, 2, 2, 0, 3, 61, 0, 0, 2, 4072, 0, 0, 0, 0, 3, 61, 0, 26, 0, 33, 143, 0, 11, 0, 0, 171, 538, 0, 26, 268, 3677, 1, 0, 0, 57, 0, 4033, 6, 0, 0, 0, 4084, 39, 4046, 3474, 0, 0, 2, 13, 0, 4, 0, 67, 10, 0, 0] payload_rows_per_dst=[7802, 280, 8144, 8192, 8012, 8038, 8160, 7996] dup_rows_per_dst=[2, 0, 132, 402, 1350, 10, 6962, 54] combine_rows_per_src=[7802, 280, 8144, 8192, 8012, 8038, 8160, 7996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55378 payload_rows_remote=48968 dup_rows=10158 dup_rows_remote=10158 dedup_ratio=0.154999 router_tokens_per_expert=[0, 0, 0, 3949, 0, 4096, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 6, 1, 0, 4055, 194, 0, 0, 0, 0, 0, 45, 0, 0, 1, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 5, 0, 0, 3200, 0, 0, 1, 0, 4035, 7, 2280, 0, 0, 1298, 0, 46, 0, 374, 919, 0] payload_rows_per_dst=[0, 8152, 8112, 8184, 8192, 6410, 8136, 8192] dup_rows_per_dst=[0, 0, 388, 62, 218, 0, 4510, 4980] combine_rows_per_src=[0, 8152, 8112, 8184, 8192, 6410, 8136, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5331552968 prefetch_bytes=452984832 reduce_bytes=201326592 weight_movement_bytes=654311424 egress_bytes=2667457224 ingress_bytes=3318407168 link_bytes=3318407168 total_nvlink_bytes=5985864392 weight_to_token_ratio=0.122724 prefetch_share_of_ingress=0.136507 dedup_ratio=0.258465 prefetch_slots=36 unused_prefetch_slots=220 slot_utilization=0.140625 reduce_slots=8 remote_demand=18 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039062 gemm_inner_pad_upper_bound=18669 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.578077771e-04 prefetch_bytes_per_gemm_flop=2.154181985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=40586 payload_rows_remote=32394 dup_rows=24950 dup_rows_remote=6984 dedup_ratio=0.380707 router_tokens_per_expert=[118, 0, 1722, 3962, 0, 3982, 0, 0, 0, 0, 0, 3291, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 3295, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 9, 4092, 0, 0, 0, 0, 3, 0, 0, 0, 0, 80, 0] payload_rows_per_dst=[8192, 8192, 5994, 8172, 8, 8054, 1808, 166] dup_rows_per_dst=[17966, 2628, 0, 4354, 2, 0, 0, 0] combine_rows_per_src=[8192, 8192, 5994, 8172, 8, 8054, 1808, 166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6123750992 prefetch_bytes=478150656 reduce_bytes=176160768 weight_movement_bytes=654311424 egress_bytes=3063670352 ingress_bytes=3714392064 link_bytes=3714392064 total_nvlink_bytes=6778062416 weight_to_token_ratio=0.106848 prefetch_share_of_ingress=0.128729 dedup_ratio=0.179825 prefetch_slots=38 unused_prefetch_slots=218 slot_utilization=0.148438 reduce_slots=7 remote_demand=19 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039292 gemm_inner_pad_upper_bound=19685 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.766389492e-04 prefetch_bytes_per_gemm_flop=2.273858762e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5753870208 prefetch_bytes=377487360 reduce_bytes=327155712 weight_movement_bytes=704643072 egress_bytes=2878723968 ingress_bytes=3579789312 link_bytes=3579789312 total_nvlink_bytes=6458513280 weight_to_token_ratio=0.122464 prefetch_share_of_ingress=0.105450 dedup_ratio=0.221479 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=13 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=17907 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.702378778e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=187604992 dispatch_meta_bytes=228544 combine_payload_bytes=187604992 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=187833536 ingress_bytes=263102464 link_bytes=263102464 token_bytes=375438528 weight_movement_bytes=75497472 total_nvlink_bytes=450936000 weight_to_token_ratio=0.201091 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.001905753e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=198508544 dispatch_meta_bytes=201656 combine_payload_bytes=198508544 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=198710200 ingress_bytes=299171840 link_bytes=299171840 token_bytes=397218744 weight_movement_bytes=100663296 total_nvlink_bytes=497882040 weight_to_token_ratio=0.253420 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.276352028e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=200572928 dispatch_meta_bytes=236504 combine_payload_bytes=200572928 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200809432 ingress_bytes=225738752 link_bytes=225738752 token_bytes=401382360 weight_movement_bytes=25165824 total_nvlink_bytes=426548184 weight_to_token_ratio=0.062698 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.717611076e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=132685824 dispatch_meta_bytes=157512 combine_payload_bytes=132685824 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=132843336 ingress_bytes=208183296 link_bytes=208183296 token_bytes=265529160 weight_movement_bytes=75497472 total_nvlink_bytes=341026632 weight_to_token_ratio=0.284328 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.584034340e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6095266520 prefetch_bytes=377487360 reduce_bytes=478150656 weight_movement_bytes=855638016 egress_bytes=3049456344 ingress_bytes=3901448192 link_bytes=3901448192 total_nvlink_bytes=6950904536 weight_to_token_ratio=0.140377 prefetch_share_of_ingress=0.096756 dedup_ratio=0.193880 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=19 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=18796 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.855344554e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6155829056 prefetch_bytes=402653184 reduce_bytes=377487360 weight_movement_bytes=780140544 egress_bytes=3079749440 ingress_bytes=3856220160 link_bytes=3856220160 total_nvlink_bytes=6935969600 weight_to_token_ratio=0.126732 prefetch_share_of_ingress=0.104417 dedup_ratio=0.178705 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=15 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=18669 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.833836238e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5016042664 prefetch_bytes=327155712 reduce_bytes=578813952 weight_movement_bytes=905969664 egress_bytes=2509651112 ingress_bytes=3412361216 link_bytes=3412361216 total_nvlink_bytes=5922012328 weight_to_token_ratio=0.180614 prefetch_share_of_ingress=0.095874 dedup_ratio=0.294575 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=23 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=18288 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.622757881e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49712 payload_rows_remote=41522 dup_rows=15824 dup_rows_remote=15572 dedup_ratio=0.241455 router_tokens_per_expert=[198, 21, 0, 4082, 0, 4072, 0, 33, 0, 0, 0, 4062, 0, 0, 19, 25, 0, 1, 3472, 0, 0, 2, 0, 0, 0, 1, 46, 0, 3271, 0, 3806, 0, 0, 0, 0, 6, 0, 4, 1028, 0, 0, 0, 1, 0, 0, 20, 27, 4056, 111, 0, 0, 0, 4089, 0, 0, 21, 0, 0, 0, 0, 0, 8, 286, 0] payload_rows_per_dst=[504, 8170, 534, 7810, 8156, 8170, 8190, 8178] dup_rows_per_dst=[0, 42, 0, 6438, 2064, 6454, 252, 574] combine_rows_per_src=[504, 8170, 534, 7810, 8156, 8170, 8190, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=170074112 dispatch_meta_bytes=228376 combine_payload_bytes=170074112 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=170302488 ingress_bytes=220405760 link_bytes=220405760 token_bytes=340376600 weight_movement_bytes=50331648 total_nvlink_bytes=390708248 weight_to_token_ratio=0.147870 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=28288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.677033169e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5537175104 prefetch_bytes=352321536 reduce_bytes=578813952 weight_movement_bytes=931135488 egress_bytes=2770343488 ingress_bytes=3697967104 link_bytes=3697967104 total_nvlink_bytes=6468310592 weight_to_token_ratio=0.168161 prefetch_share_of_ingress=0.095274 dedup_ratio=0.243280 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=23 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=17653 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.758578556e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none + [2026-09-21 08:31:45] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 221.8 | throughput per GPU (TFLOP/s/GPU): 145.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.023766E+01 | loss scale: 1.0 | grad norm: 28.792 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46414 payload_rows_remote=40398 dup_rows=19122 dup_rows_remote=15874 dedup_ratio=0.291779 router_tokens_per_expert=[1495, 711, 86, 61, 34, 951, 1013, 207, 53, 863, 61, 12, 37, 111, 174, 531, 40, 439, 580, 1825, 21, 709, 276, 38, 978, 51, 70, 326, 530, 1644, 406, 627, 207, 242, 655, 163, 259, 1729, 243, 438, 484, 1177, 813, 26, 2042, 1265, 36, 687, 900, 485, 962, 117, 1095, 742, 627, 150, 964, 27, 755, 108, 19, 122, 172, 97] payload_rows_per_dst=[6056, 5648, 6414, 6016, 5778, 5472, 4664, 6366] dup_rows_per_dst=[3060, 2026, 3150, 3248, 2094, 1796, 1502, 2246] combine_rows_per_src=[6056, 5648, 6414, 6016, 5778, 5472, 4664, 6366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46106 payload_rows_remote=39866 dup_rows=19430 dup_rows_remote=16194 dedup_ratio=0.296478 router_tokens_per_expert=[1785, 704, 183, 26, 25, 722, 1219, 74, 21, 1362, 97, 3, 17, 146, 140, 618, 129, 276, 778, 1249, 59, 443, 401, 79, 578, 129, 26, 678, 561, 981, 277, 741, 305, 354, 506, 319, 218, 1843, 517, 635, 367, 1502, 638, 19, 1782, 1285, 41, 442, 800, 870, 914, 86, 912, 722, 616, 131, 869, 2, 512, 87, 65, 117, 711, 54] payload_rows_per_dst=[6240, 5972, 6576, 5620, 6338, 6292, 4980, 4088] dup_rows_per_dst=[3236, 2260, 3256, 2322, 3056, 2856, 1698, 746] combine_rows_per_src=[6240, 5972, 6576, 5620, 6338, 6292, 4980, 4088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46650 payload_rows_remote=40272 dup_rows=18886 dup_rows_remote=15832 dedup_ratio=0.288177 router_tokens_per_expert=[1850, 755, 136, 79, 43, 436, 1020, 117, 10, 1645, 102, 0, 45, 205, 100, 778, 52, 351, 545, 1570, 19, 641, 364, 107, 801, 45, 104, 793, 461, 741, 357, 874, 223, 305, 664, 447, 301, 1751, 237, 788, 476, 1395, 598, 20, 2108, 1014, 27, 498, 1102, 588, 415, 28, 1085, 620, 352, 92, 1005, 39, 582, 53, 15, 111, 623, 60] payload_rows_per_dst=[6172, 6664, 5582, 5826, 6378, 5910, 3536, 6582] dup_rows_per_dst=[2700, 3480, 1716, 2526, 3054, 2146, 654, 2610] combine_rows_per_src=[6172, 6664, 5582, 5826, 6378, 5910, 3536, 6582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46286 payload_rows_remote=39678 dup_rows=19250 dup_rows_remote=15532 dedup_ratio=0.293732 router_tokens_per_expert=[1952, 622, 194, 33, 43, 1017, 884, 132, 151, 1322, 21, 1, 31, 227, 180, 301, 87, 639, 874, 1875, 54, 379, 287, 134, 628, 116, 44, 469, 604, 835, 309, 411, 337, 265, 796, 455, 188, 1700, 199, 873, 315, 834, 681, 9, 1388, 891, 23, 539, 1073, 387, 1198, 115, 956, 667, 1227, 165, 1175, 41, 596, 10, 83, 220, 420, 86] payload_rows_per_dst=[6562, 6136, 6608, 5066, 6342, 4094, 5488, 5990] dup_rows_per_dst=[3192, 2390, 3718, 1766, 3284, 822, 2030, 2048] combine_rows_per_src=[6562, 6136, 6608, 5066, 6342, 4094, 5488, 5990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46594 payload_rows_remote=40580 dup_rows=18942 dup_rows_remote=16578 dedup_ratio=0.289032 router_tokens_per_expert=[1998, 778, 107, 39, 19, 861, 920, 79, 11, 1449, 37, 1, 48, 63, 53, 675, 64, 538, 352, 1607, 38, 439, 397, 115, 700, 57, 49, 891, 843, 1090, 175, 535, 243, 349, 503, 366, 172, 1475, 287, 644, 514, 1062, 900, 5, 1767, 1238, 21, 449, 1038, 480, 738, 110, 1268, 1071, 695, 246, 772, 2, 553, 39, 43, 258, 366, 66] payload_rows_per_dst=[5108, 5674, 5374, 6040, 5902, 6014, 6532, 5950] dup_rows_per_dst=[1976, 1954, 1726, 2894, 2176, 2364, 4070, 1782] combine_rows_per_src=[5108, 5674, 5374, 6040, 5902, 6014, 6532, 5950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45262 payload_rows_remote=38430 dup_rows=20274 dup_rows_remote=17286 dedup_ratio=0.309357 router_tokens_per_expert=[1929, 736, 119, 80, 29, 689, 950, 79, 160, 963, 63, 1, 29, 160, 101, 921, 103, 497, 580, 1416, 41, 434, 288, 34, 859, 69, 127, 890, 476, 1125, 238, 821, 108, 249, 656, 261, 176, 1414, 270, 793, 476, 1632, 559, 22, 2307, 1056, 42, 380, 1023, 532, 830, 44, 922, 824, 467, 115, 1324, 10, 614, 18, 4, 67, 482, 84] payload_rows_per_dst=[4252, 5184, 5208, 5730, 4492, 6830, 6734, 6832] dup_rows_per_dst=[1112, 1456, 1578, 2638, 1376, 4332, 4794, 2988] combine_rows_per_src=[4252, 5184, 5208, 5730, 4492, 6830, 6734, 6832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46452 payload_rows_remote=39936 dup_rows=19084 dup_rows_remote=16134 dedup_ratio=0.291199 router_tokens_per_expert=[1827, 737, 194, 53, 18, 757, 1418, 78, 14, 1917, 23, 1, 36, 179, 283, 529, 65, 345, 532, 1182, 69, 430, 487, 39, 708, 180, 20, 742, 518, 753, 201, 648, 306, 370, 580, 322, 304, 1185, 225, 823, 520, 804, 1009, 17, 2110, 1099, 32, 531, 644, 593, 855, 153, 1107, 838, 729, 48, 1067, 6, 706, 29, 20, 160, 567, 26] payload_rows_per_dst=[6468, 6516, 5662, 5430, 5882, 5422, 4812, 6260] dup_rows_per_dst=[3696, 2950, 2244, 2110, 2348, 1752, 1620, 2364] combine_rows_per_src=[6468, 6516, 5662, 5430, 5882, 5422, 4812, 6260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165470208 dispatch_meta_bytes=225088 combine_payload_bytes=165470208 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=165695296 ingress_bytes=215801856 link_bytes=215801856 token_bytes=331165504 weight_movement_bytes=50331648 total_nvlink_bytes=381497152 weight_to_token_ratio=0.151983 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.642002779e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163291136 dispatch_meta_bytes=224240 combine_payload_bytes=163291136 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=163515376 ingress_bytes=213622784 link_bytes=213622784 token_bytes=326806512 weight_movement_bytes=50331648 total_nvlink_bytes=377138160 weight_to_token_ratio=0.154011 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.625422559e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164954112 dispatch_meta_bytes=224416 combine_payload_bytes=164954112 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=165178528 ingress_bytes=215285760 link_bytes=215285760 token_bytes=330132640 weight_movement_bytes=50331648 total_nvlink_bytes=380464288 weight_to_token_ratio=0.152459 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.638075885e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162521088 dispatch_meta_bytes=220840 combine_payload_bytes=162521088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=162741928 ingress_bytes=187686912 link_bytes=187686912 token_bytes=325263016 weight_movement_bytes=25165824 total_nvlink_bytes=350428840 weight_to_token_ratio=0.077371 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.428080540e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166215680 dispatch_meta_bytes=228632 combine_payload_bytes=166215680 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=166444312 ingress_bytes=241713152 link_bytes=241713152 token_bytes=332659992 weight_movement_bytes=75497472 total_nvlink_bytes=408157464 weight_to_token_ratio=0.226951 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.839157803e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=157409280 dispatch_meta_bytes=222864 combine_payload_bytes=157409280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=157632144 ingress_bytes=182575104 link_bytes=182575104 token_bytes=315041424 weight_movement_bytes=25165824 total_nvlink_bytes=340207248 weight_to_token_ratio=0.079881 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.389185588e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46156 payload_rows_remote=39078 dup_rows=19380 dup_rows_remote=14036 dedup_ratio=0.295715 router_tokens_per_expert=[2484, 772, 241, 27, 26, 810, 918, 59, 81, 1269, 39, 1, 58, 174, 128, 722, 44, 476, 608, 1477, 37, 336, 321, 63, 531, 44, 45, 833, 789, 875, 371, 664, 186, 286, 688, 143, 164, 1289, 292, 840, 631, 1428, 891, 14, 2321, 974, 20, 389, 925, 519, 945, 122, 1141, 775, 391, 50, 714, 14, 583, 97, 7, 325, 240, 41] payload_rows_per_dst=[4508, 5394, 5220, 5842, 5192, 6450, 7078, 6472] dup_rows_per_dst=[1198, 1832, 1504, 2462, 1844, 2984, 5344, 2212] combine_rows_per_src=[4508, 5394, 5220, 5842, 5192, 6450, 7078, 6472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163577856 dispatch_meta_bytes=224280 combine_payload_bytes=163577856 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=163802136 ingress_bytes=213909504 link_bytes=213909504 token_bytes=327379992 weight_movement_bytes=50331648 total_nvlink_bytes=377711640 weight_to_token_ratio=0.153741 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=10880 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.627604167e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50836 payload_rows_remote=43366 dup_rows=14700 dup_rows_remote=11408 dedup_ratio=0.224304 router_tokens_per_expert=[27, 2147, 13, 6, 21, 0, 117, 0, 2, 20, 3, 0, 3783, 1, 4, 0, 2162, 0, 119, 0, 5, 3047, 0, 2246, 0, 357, 30, 35, 2243, 0, 3, 154, 3568, 0, 7, 0, 0, 0, 208, 1112, 0, 768, 430, 0, 3978, 0, 0, 0, 1372, 922, 0, 1151, 2, 0, 0, 2559, 0, 0, 91, 0, 52, 0, 3, 0] payload_rows_per_dst=[8132, 7614, 7484, 7470, 7584, 2384, 5580, 4588] dup_rows_per_dst=[4448, 12, 3350, 3292, 2244, 12, 1314, 28] combine_rows_per_src=[8132, 7614, 7484, 7470, 7584, 2384, 5580, 4588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=212456 combine_payload_bytes=160063488 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160275944 ingress_bytes=235560960 link_bytes=235560960 token_bytes=320339432 weight_movement_bytes=75497472 total_nvlink_bytes=395836904 weight_to_token_ratio=0.235680 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=6912 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792346730e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43844 payload_rows_remote=35676 dup_rows=21692 dup_rows_remote=12520 dedup_ratio=0.330994 router_tokens_per_expert=[40, 1964, 2436, 0, 0, 0, 150, 1, 187, 1, 32, 0, 3985, 7, 1, 1, 2727, 0, 134, 4, 7, 609, 12, 3993, 3, 11, 1, 0, 1168, 26, 0, 4, 3582, 0, 1, 0, 0, 0, 181, 1145, 14, 744, 93, 0, 4079, 1, 0, 1, 768, 499, 0, 1, 0, 1, 0, 4083, 1, 0, 53, 3, 2, 0, 12, 0] payload_rows_per_dst=[8168, 8188, 8124, 7192, 2554, 1688, 2386, 5544] dup_rows_per_dst=[9172, 1304, 8558, 2336, 100, 18, 152, 52] combine_rows_per_src=[8168, 8188, 8124, 7192, 2554, 1688, 2386, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50588 payload_rows_remote=42422 dup_rows=14948 dup_rows_remote=6908 dedup_ratio=0.228088 router_tokens_per_expert=[0, 784, 410, 0, 30, 6, 344, 4, 0, 60, 0, 2, 4020, 2, 3, 0, 3896, 2, 51, 0, 0, 1234, 0, 3295, 0, 0, 0, 4, 1127, 0, 0, 1, 3634, 0, 50, 0, 0, 0, 2, 750, 34, 1173, 1396, 27, 4079, 0, 1, 8, 2599, 273, 2, 0, 0, 38, 0, 3329, 49, 0, 23, 26, 0, 0, 0, 0] payload_rows_per_dst=[2946, 8108, 6630, 3616, 8166, 6680, 6504, 7938] dup_rows_per_dst=[210, 66, 1738, 10, 8040, 2354, 1684, 846] combine_rows_per_src=[2946, 8108, 6630, 3616, 8166, 6680, 6504, 7938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52264 payload_rows_remote=44078 dup_rows=13272 dup_rows_remote=6880 dedup_ratio=0.202515 router_tokens_per_expert=[1, 193, 289, 0, 0, 64, 1011, 0, 0, 6, 0, 0, 4070, 0, 1, 0, 1260, 5, 0, 0, 0, 3121, 0, 3145, 4, 162, 0, 1, 2006, 0, 6, 7, 2428, 0, 25, 1, 0, 0, 131, 2045, 171, 2913, 18, 0, 3964, 0, 219, 4, 277, 1098, 33, 4, 0, 0, 0, 3895, 0, 0, 17, 0, 173, 0, 0, 0] payload_rows_per_dst=[2974, 8150, 6252, 4296, 7350, 8186, 7878, 7178] dup_rows_per_dst=[142, 4, 0, 76, 1910, 6392, 2736, 2012] combine_rows_per_src=[2974, 8150, 6252, 4296, 7350, 8186, 7878, 7178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52740 payload_rows_remote=46688 dup_rows=12796 dup_rows_remote=11892 dedup_ratio=0.195251 router_tokens_per_expert=[0, 3158, 152, 0, 0, 0, 27, 0, 0, 0, 57, 0, 3992, 0, 28, 0, 2649, 0, 5, 0, 0, 2623, 7, 732, 0, 0, 0, 1, 3475, 0, 0, 2, 978, 0, 0, 0, 0, 0, 428, 1533, 14, 3225, 2, 0, 4091, 0, 15, 0, 2031, 1569, 0, 5, 0, 2, 0, 1955, 9, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[8182, 8058, 6052, 7800, 4826, 6498, 6018, 5306] dup_rows_per_dst=[6674, 96, 904, 3066, 830, 14, 1196, 16] combine_rows_per_src=[8182, 8058, 6052, 7800, 4826, 6498, 6018, 5306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51078 payload_rows_remote=43100 dup_rows=14458 dup_rows_remote=10980 dedup_ratio=0.220612 router_tokens_per_expert=[6, 1755, 101, 0, 0, 0, 0, 0, 26, 0, 2, 47, 3957, 0, 0, 7, 3825, 4, 5, 0, 0, 438, 9, 1841, 1, 9, 0, 16, 3832, 0, 1, 3, 2659, 6, 13, 0, 3, 0, 334, 1142, 13, 3077, 58, 4, 4089, 0, 0, 1, 1064, 610, 1, 22, 0, 0, 0, 3725, 31, 0, 28, 0, 3, 0, 0, 0] payload_rows_per_dst=[3602, 7938, 904, 7668, 6914, 8190, 7884, 7978] dup_rows_per_dst=[122, 140, 8, 56, 1400, 6294, 2960, 3478] combine_rows_per_src=[3602, 7938, 904, 7668, 6914, 8190, 7884, 7978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51858 payload_rows_remote=44268 dup_rows=13678 dup_rows_remote=13586 dedup_ratio=0.208710 router_tokens_per_expert=[0, 211, 353, 0, 0, 1, 494, 0, 0, 0, 115, 2, 3624, 0, 100, 0, 3156, 13, 22, 33, 0, 1186, 0, 844, 0, 127, 1, 2, 3106, 0, 5, 86, 4096, 0, 2, 0, 0, 1, 142, 1261, 3, 2494, 197, 2, 3930, 0, 1, 0, 1880, 1163, 39, 3, 0, 0, 0, 4049, 0, 0, 16, 0, 0, 0, 8, 0] payload_rows_per_dst=[8184, 7590, 8192, 8178, 2684, 5364, 5340, 6326] dup_rows_per_dst=[1794, 92, 4196, 6574, 128, 30, 830, 34] combine_rows_per_src=[8184, 7590, 8192, 8178, 2684, 5364, 5340, 6326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=177627136 dispatch_meta_bytes=219096 combine_payload_bytes=177627136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=177846232 ingress_bytes=202792960 link_bytes=202792960 token_bytes=355473368 weight_movement_bytes=25165824 total_nvlink_bytes=380639192 weight_to_token_ratio=0.070795 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.543020111e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=146128896 dispatch_meta_bytes=192784 combine_payload_bytes=146128896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=146321680 ingress_bytes=171294720 link_bytes=171294720 token_bytes=292450576 weight_movement_bytes=25165824 total_nvlink_bytes=317616400 weight_to_token_ratio=0.086052 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.303354899e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=173760512 dispatch_meta_bytes=197320 combine_payload_bytes=173760512 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=173957832 ingress_bytes=224092160 link_bytes=224092160 token_bytes=347718344 weight_movement_bytes=50331648 total_nvlink_bytes=398049992 weight_to_token_ratio=0.144748 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.705082413e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=180543488 dispatch_meta_bytes=203832 combine_payload_bytes=180543488 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=180747320 ingress_bytes=256040960 link_bytes=256040960 token_bytes=361290808 weight_movement_bytes=75497472 total_nvlink_bytes=436788280 weight_to_token_ratio=0.208966 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.948175867e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=191234048 dispatch_meta_bytes=234320 combine_payload_bytes=191234048 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=191468368 ingress_bytes=266731520 link_bytes=266731520 token_bytes=382702416 weight_movement_bytes=75497472 total_nvlink_bytes=458199888 weight_to_token_ratio=0.197275 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=13952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.029518676e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=176537600 dispatch_meta_bytes=216320 combine_payload_bytes=176537600 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=176753920 ingress_bytes=226869248 link_bytes=226869248 token_bytes=353291520 weight_movement_bytes=50331648 total_nvlink_bytes=403623168 weight_to_token_ratio=0.142465 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.726212844e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=181321728 dispatch_meta_bytes=231416 combine_payload_bytes=181321728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=181553144 ingress_bytes=206487552 link_bytes=206487552 token_bytes=362874872 weight_movement_bytes=25165824 total_nvlink_bytes=388040696 weight_to_token_ratio=0.069351 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.571131688e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48450 payload_rows_remote=40264 dup_rows=17086 dup_rows_remote=12198 dedup_ratio=0.260712 router_tokens_per_expert=[12, 341, 38, 0, 0, 0, 2, 0, 8, 0, 0, 22, 4058, 0, 0, 0, 3153, 0, 0, 0, 4, 1567, 1, 3924, 0, 0, 0, 0, 1940, 0, 0, 13, 4050, 0, 1, 0, 0, 0, 4, 1348, 36, 1061, 579, 0, 4027, 0, 13, 0, 2389, 43, 0, 26, 0, 0, 0, 4079, 0, 0, 4, 0, 2, 0, 23, 0] payload_rows_per_dst=[774, 8130, 3144, 3892, 8170, 8186, 8186, 7968] dup_rows_per_dst=[12, 46, 0, 14, 2636, 3246, 4888, 6244] combine_rows_per_src=[774, 8130, 3144, 3892, 8170, 8186, 8186, 7968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53718 payload_rows_remote=45662 dup_rows=11818 dup_rows_remote=8980 dedup_ratio=0.180328 router_tokens_per_expert=[11, 0, 110, 0, 0, 0, 3, 3, 229, 1037, 0, 682, 0, 0, 0, 0, 0, 1285, 19, 1001, 0, 0, 0, 2, 0, 0, 0, 0, 3495, 1112, 0, 840, 2, 17, 0, 4, 4075, 0, 161, 7, 0, 0, 0, 257, 0, 3934, 2547, 0, 30, 2631, 16, 4074, 0, 0, 0, 1433, 5, 0, 22, 0, 0, 0, 3722, 2] payload_rows_per_dst=[8176, 5170, 2042, 8056, 8164, 7910, 6704, 7496] dup_rows_per_dst=[226, 1296, 2, 2838, 368, 5566, 1516, 6] combine_rows_per_src=[8176, 5170, 2042, 8056, 8164, 7910, 6704, 7496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55270 payload_rows_remote=47086 dup_rows=10266 dup_rows_remote=10242 dedup_ratio=0.156647 router_tokens_per_expert=[9, 0, 7, 1, 0, 0, 0, 2, 1, 1263, 0, 19, 0, 0, 0, 1, 395, 2451, 7, 1582, 73, 0, 0, 0, 0, 1, 33, 0, 4080, 2144, 0, 0, 0, 0, 804, 4, 3798, 0, 18, 1, 0, 0, 0, 3, 0, 2851, 0, 0, 2, 3437, 0, 4085, 0, 1, 23, 1420, 0, 157, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8184, 6620, 8192, 5314, 8150, 3214, 7406, 8190] dup_rows_per_dst=[24, 850, 4082, 590, 2046, 0, 2360, 314] combine_rows_per_src=[8184, 6620, 8192, 5314, 8150, 3214, 7406, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54946 payload_rows_remote=47148 dup_rows=10590 dup_rows_remote=10306 dedup_ratio=0.161591 router_tokens_per_expert=[0, 0, 0, 5, 1, 2, 4, 15, 0, 101, 0, 933, 0, 10, 0, 3, 766, 3677, 79, 2978, 0, 0, 0, 0, 0, 0, 652, 0, 4074, 3111, 0, 1, 0, 0, 28, 109, 3816, 37, 51, 0, 0, 0, 0, 24, 114, 4096, 1, 0, 59, 676, 0, 3976, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 3365, 0] payload_rows_per_dst=[7992, 7782, 6842, 8154, 7798, 8192, 1452, 6734] dup_rows_per_dst=[14, 1666, 804, 7522, 284, 278, 22, 0] combine_rows_per_src=[7992, 7782, 6842, 8154, 7798, 8192, 1452, 6734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63670 payload_rows_remote=56194 dup_rows=1866 dup_rows_remote=1802 dedup_ratio=0.028473 router_tokens_per_expert=[7, 1, 0, 0, 101, 0, 0, 0, 39, 0, 0, 208, 5, 0, 0, 0, 7, 3854, 4, 4001, 0, 0, 0, 0, 0, 0, 28, 1, 3688, 116, 0, 119, 0, 2, 0, 0, 4084, 3, 0, 0, 0, 0, 0, 116, 22, 3630, 1, 1, 0, 3984, 3, 4030, 47, 0, 21, 13, 35, 488, 0, 6, 0, 19, 4084, 0] payload_rows_per_dst=[8166, 7936, 8014, 7732, 8172, 7476, 7996, 8178] dup_rows_per_dst=[112, 276, 10, 172, 6, 64, 140, 1086] combine_rows_per_src=[8166, 7936, 8014, 7732, 8172, 7476, 7996, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54826 payload_rows_remote=47638 dup_rows=10710 dup_rows_remote=7340 dedup_ratio=0.163422 router_tokens_per_expert=[1593, 0, 64, 0, 3, 0, 0, 0, 51, 2792, 0, 0, 27, 0, 11, 14, 166, 2384, 20, 1851, 1126, 0, 0, 0, 0, 0, 0, 0, 1337, 160, 1, 122, 1, 0, 743, 2, 4076, 1, 118, 3, 0, 0, 0, 0, 7, 3975, 32, 1, 0, 4071, 0, 3975, 0, 29, 4, 5, 0, 0, 0, 0, 0, 0, 4003, 0] payload_rows_per_dst=[8118, 7188, 6666, 566, 8168, 7964, 8150, 8006] dup_rows_per_dst=[3152, 3370, 2334, 0, 1720, 66, 68, 0] combine_rows_per_src=[8118, 7188, 6666, 566, 8168, 7964, 8150, 8006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56106 payload_rows_remote=55424 dup_rows=9430 dup_rows_remote=9070 dedup_ratio=0.143890 router_tokens_per_expert=[207, 0, 0, 0, 24, 0, 0, 0, 0, 1024, 0, 63, 0, 0, 0, 0, 19, 3642, 30, 108, 142, 0, 0, 0, 0, 0, 18, 0, 3878, 2162, 0, 0, 7, 0, 584, 1, 4012, 0, 36, 0, 0, 0, 0, 1324, 1, 3448, 16, 0, 0, 3867, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0] payload_rows_per_dst=[8184, 8042, 682, 7570, 8190, 7514, 7734, 8190] dup_rows_per_dst=[394, 1416, 360, 4102, 1090, 2064, 0, 4] combine_rows_per_src=[8184, 8042, 682, 7570, 8190, 7514, 7734, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=164921344 dispatch_meta_bytes=209848 combine_payload_bytes=164921344 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=165131192 ingress_bytes=240418816 link_bytes=240418816 token_bytes=330052536 weight_movement_bytes=75497472 total_nvlink_bytes=405550008 weight_to_token_ratio=0.228744 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=12928 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.829309401e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46386 payload_rows_remote=38202 dup_rows=19150 dup_rows_remote=18076 dedup_ratio=0.292206 router_tokens_per_expert=[122, 1, 0, 0, 0, 0, 0, 4, 0, 1049, 0, 1611, 4, 0, 0, 0, 5, 3171, 0, 4064, 1018, 0, 0, 0, 0, 0, 119, 0, 1844, 164, 0, 6, 0, 0, 358, 0, 4083, 0, 3, 0, 0, 0, 0, 3, 5, 3956, 0, 0, 4, 2939, 0, 4050, 0, 0, 1, 91, 0, 4, 0, 0, 0, 0, 4089, 0] payload_rows_per_dst=[4684, 4860, 8188, 4100, 3302, 6854, 6214, 8184] dup_rows_per_dst=[218, 468, 8328, 166, 698, 2, 8196, 1074] combine_rows_per_src=[4684, 4860, 8188, 4100, 3302, 6854, 6214, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=187031552 dispatch_meta_bytes=218568 combine_payload_bytes=187031552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=187250120 ingress_bytes=237363200 link_bytes=237363200 token_bytes=374281672 weight_movement_bytes=50331648 total_nvlink_bytes=424613320 weight_to_token_ratio=0.134475 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.806059694e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=192864256 dispatch_meta_bytes=229312 combine_payload_bytes=192864256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193093568 ingress_bytes=218030080 link_bytes=218030080 token_bytes=385957824 weight_movement_bytes=25165824 total_nvlink_bytes=411123648 weight_to_token_ratio=0.065204 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.658956989e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193118208 dispatch_meta_bytes=229816 combine_payload_bytes=193118208 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=193348024 ingress_bytes=243449856 link_bytes=243449856 token_bytes=386466232 weight_movement_bytes=50331648 total_nvlink_bytes=436797880 weight_to_token_ratio=0.130236 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.852372113e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=230170624 dispatch_meta_bytes=231984 combine_payload_bytes=230170624 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=230402608 ingress_bytes=305668096 link_bytes=305668096 token_bytes=460573232 weight_movement_bytes=75497472 total_nvlink_bytes=536070704 weight_to_token_ratio=0.163921 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.325781031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=195125248 dispatch_meta_bytes=219912 combine_payload_bytes=195125248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=195345160 ingress_bytes=220291072 link_bytes=220291072 token_bytes=390470408 weight_movement_bytes=25165824 total_nvlink_bytes=415636232 weight_to_token_ratio=0.064450 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.676160526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=227016704 dispatch_meta_bytes=257976 combine_payload_bytes=227016704 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=227274680 ingress_bytes=277348352 link_bytes=277348352 token_bytes=454291384 weight_movement_bytes=50331648 total_nvlink_bytes=504623032 weight_to_token_ratio=0.110792 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.110300500e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156475392 dispatch_meta_bytes=225112 combine_payload_bytes=156475392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=156700504 ingress_bytes=181641216 link_bytes=181641216 token_bytes=313175896 weight_movement_bytes=25165824 total_nvlink_bytes=338341720 weight_to_token_ratio=0.080357 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.382079779e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56850 payload_rows_remote=48696 dup_rows=8686 dup_rows_remote=1208 dedup_ratio=0.132538 router_tokens_per_expert=[4, 0, 185, 0, 4088, 60, 4096, 3737, 2, 0, 0, 0, 0, 7, 0, 0, 3921, 0, 0, 0, 147, 0, 0, 0, 2147, 0, 0, 0, 0, 2458, 0, 10, 2317, 9, 0, 433, 0, 0, 0, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 1, 0, 0, 0, 0, 2821, 3201, 209, 0, 20] payload_rows_per_dst=[7562, 8180, 7996, 8154, 5088, 8192, 5788, 5890] dup_rows_per_dst=[410, 14, 140, 7478, 430, 0, 4, 210] combine_rows_per_src=[7562, 8180, 7996, 8154, 5088, 8192, 5788, 5890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60944 payload_rows_remote=53508 dup_rows=4592 dup_rows_remote=4450 dedup_ratio=0.070068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 61, 2, 0, 0, 0, 0, 120, 2868, 0, 3510, 2, 0, 0, 0, 0, 0, 100, 0, 4094, 131, 0, 2, 0, 0, 3, 0, 3687, 0, 0, 0, 0, 0, 0, 206, 0, 3974, 6, 0, 2, 3709, 0, 3946, 0, 0, 77, 1, 0, 430, 0, 0, 0, 0, 3607, 0] payload_rows_per_dst=[7892, 6900, 7596, 8190, 7380, 8144, 7436, 7406] dup_rows_per_dst=[0, 1696, 1394, 464, 0, 228, 142, 668] combine_rows_per_src=[7892, 6900, 7596, 8190, 7380, 8144, 7436, 7406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59914 payload_rows_remote=54038 dup_rows=5622 dup_rows_remote=4854 dedup_ratio=0.085785 router_tokens_per_expert=[0, 0, 1, 0, 4081, 26, 4095, 1676, 0, 0, 0, 0, 16, 3, 0, 20, 4086, 0, 0, 0, 40, 0, 5, 3, 40, 0, 0, 0, 0, 0, 0, 0, 3838, 1, 0, 2038, 0, 0, 0, 4069, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3908, 3905, 532, 0, 377] payload_rows_per_dst=[5876, 7954, 6140, 7814, 8080, 8190, 8140, 7720] dup_rows_per_dst=[768, 26, 72, 76, 3674, 12, 0, 994] combine_rows_per_src=[5876, 7954, 6140, 7814, 8080, 8190, 8140, 7720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62660 payload_rows_remote=56660 dup_rows=2876 dup_rows_remote=2390 dedup_ratio=0.043884 router_tokens_per_expert=[0, 0, 210, 0, 3915, 5, 4093, 3707, 0, 21, 178, 0, 0, 0, 0, 0, 4062, 0, 905, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 70, 8, 3, 459, 0, 0, 2782, 0, 0, 2, 4034, 56, 1, 0, 9, 0, 0, 2, 0, 0, 41, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4087, 18, 0, 0] payload_rows_per_dst=[7574, 8186, 8180, 8188, 6000, 8192, 8150, 8190] dup_rows_per_dst=[270, 42, 1756, 154, 486, 130, 0, 38] combine_rows_per_src=[7574, 8186, 8180, 8188, 6000, 8192, 8150, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=199458816 dispatch_meta_bytes=199616 combine_payload_bytes=199458816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199658432 ingress_bytes=224624640 link_bytes=224624640 token_bytes=399117248 weight_movement_bytes=25165824 total_nvlink_bytes=424283072 weight_to_token_ratio=0.063054 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=24832 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.709133971e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58780 payload_rows_remote=50614 dup_rows=6756 dup_rows_remote=6754 dedup_ratio=0.103088 router_tokens_per_expert=[0, 0, 908, 2, 4094, 932, 4083, 3843, 0, 62, 0, 0, 0, 0, 0, 8, 4081, 0, 1, 4, 0, 0, 1, 0, 7, 0, 0, 16, 0, 539, 0, 18, 3764, 0, 0, 1064, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 4068, 14, 0, 3] payload_rows_per_dst=[7848, 8188, 8162, 8162, 7710, 8166, 8126, 2418] dup_rows_per_dst=[3522, 140, 12, 1134, 1946, 2, 0, 0] combine_rows_per_src=[7848, 8188, 8162, 8162, 7710, 8166, 8126, 2418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59176 payload_rows_remote=51142 dup_rows=6360 dup_rows_remote=6280 dedup_ratio=0.097046 router_tokens_per_expert=[0, 1, 0, 1, 4014, 478, 3963, 3793, 0, 40, 0, 0, 3, 0, 0, 0, 4095, 0, 0, 0, 5, 13, 0, 6, 46, 0, 9, 0, 38, 393, 0, 3, 4063, 0, 0, 2416, 0, 0, 0, 3542, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 633, 4095, 69, 1, 1039] payload_rows_per_dst=[8150, 8034, 8190, 8192, 8144, 7930, 7096, 3440] dup_rows_per_dst=[396, 80, 48, 976, 4814, 0, 2, 44] combine_rows_per_src=[8150, 8034, 8190, 8192, 8144, 7930, 7096, 3440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53860 payload_rows_remote=45724 dup_rows=11676 dup_rows_remote=7204 dedup_ratio=0.178162 router_tokens_per_expert=[21, 1, 0, 0, 4095, 0, 4092, 3378, 0, 0, 54, 0, 1, 0, 0, 2, 4096, 0, 0, 0, 50, 0, 0, 64, 0, 2, 0, 0, 0, 2, 1, 0, 3444, 0, 0, 3191, 0, 1, 0, 4059, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2101, 4068, 1, 0, 39] payload_rows_per_dst=[6580, 8192, 8192, 10, 6384, 8186, 8180, 8136] dup_rows_per_dst=[0, 112, 228, 0, 0, 220, 6644, 4472] combine_rows_per_src=[6580, 8192, 8192, 10, 6384, 8186, 8180, 8136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61092 payload_rows_remote=52900 dup_rows=4444 dup_rows_remote=4424 dedup_ratio=0.067810 router_tokens_per_expert=[2, 0, 5, 0, 4095, 184, 4092, 3588, 0, 0, 1, 0, 1, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 2, 1308, 0, 15, 0, 1, 199, 0, 0, 137, 1, 0, 3245, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 4094, 2071, 0, 4] payload_rows_per_dst=[7228, 8192, 8192, 8188, 6640, 8188, 8192, 6272] dup_rows_per_dst=[330, 2, 20, 3046, 126, 0, 0, 920] combine_rows_per_src=[7228, 8192, 8192, 8188, 6640, 8188, 8192, 6272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=221339648 dispatch_meta_bytes=235568 combine_payload_bytes=221339648 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=221575216 ingress_bytes=322002944 link_bytes=322002944 token_bytes=442914864 weight_movement_bytes=100663296 total_nvlink_bytes=543578160 weight_to_token_ratio=0.227275 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.450070350e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=219168768 dispatch_meta_bytes=231832 combine_payload_bytes=219168768 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=219400600 ingress_bytes=269500416 link_bytes=269500416 token_bytes=438569368 weight_movement_bytes=50331648 total_nvlink_bytes=488901016 weight_to_token_ratio=0.114763 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.050586775e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=232079360 dispatch_meta_bytes=236200 combine_payload_bytes=232079360 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=232315560 ingress_bytes=307576832 link_bytes=307576832 token_bytes=464394920 weight_movement_bytes=75497472 total_nvlink_bytes=539892392 weight_to_token_ratio=0.162572 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=17280 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.340304306e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=207314944 dispatch_meta_bytes=229472 combine_payload_bytes=207314944 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=207544416 ingress_bytes=257646592 link_bytes=257646592 token_bytes=414859360 weight_movement_bytes=50331648 total_nvlink_bytes=465191008 weight_to_token_ratio=0.121322 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.960392871e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=209477632 dispatch_meta_bytes=229688 combine_payload_bytes=209477632 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=209707320 ingress_bytes=234643456 link_bytes=234643456 token_bytes=419184952 weight_movement_bytes=25165824 total_nvlink_bytes=444350776 weight_to_token_ratio=0.060035 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785365585e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=216678400 dispatch_meta_bytes=229296 combine_payload_bytes=216678400 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=216907696 ingress_bytes=267010048 link_bytes=267010048 token_bytes=433586096 weight_movement_bytes=50331648 total_nvlink_bytes=483917744 weight_to_token_ratio=0.116082 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.031637952e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=187285504 dispatch_meta_bytes=211712 combine_payload_bytes=187285504 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=187497216 ingress_bytes=262782976 link_bytes=262782976 token_bytes=374782720 weight_movement_bytes=75497472 total_nvlink_bytes=450280192 weight_to_token_ratio=0.201443 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.999474818e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61180 payload_rows_remote=53696 dup_rows=4356 dup_rows_remote=4356 dedup_ratio=0.066467 router_tokens_per_expert=[0, 0, 0, 218, 0, 1, 4096, 11, 0, 0, 0, 0, 3426, 0, 0, 100, 0, 7, 4094, 0, 2923, 6, 0, 73, 0, 0, 4003, 3742, 0, 0, 0, 0, 0, 1, 1466, 0, 382, 0, 4000, 0, 1, 0, 0, 0, 71, 7, 0, 48, 4046, 0, 0, 0, 16, 0, 0, 0, 28, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8192, 6860, 6008, 7484, 8188, 8132, 8124, 8192] dup_rows_per_dst=[460, 192, 10, 0, 3510, 128, 0, 56] combine_rows_per_src=[8192, 6860, 6008, 7484, 8188, 8132, 8124, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53664 payload_rows_remote=45518 dup_rows=11872 dup_rows_remote=11872 dedup_ratio=0.181152 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4073, 0, 7, 0, 0, 0, 2324, 0, 0, 1, 21, 1, 4084, 0, 3874, 3687, 0, 0, 0, 0, 4094, 3759, 2, 0, 0, 23, 0, 0, 2, 0, 28, 0, 4092, 0, 0, 4, 0, 0, 147, 0, 3, 1, 2533, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 1, 1] payload_rows_per_dst=[8146, 8192, 7416, 8192, 8184, 294, 5068, 8172] dup_rows_per_dst=[0, 4220, 2, 7564, 60, 16, 0, 10] combine_rows_per_src=[8146, 8192, 7416, 8192, 8184, 294, 5068, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=219938816 dispatch_meta_bytes=232208 combine_payload_bytes=219938816 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=220171024 ingress_bytes=295436288 link_bytes=295436288 token_bytes=440109840 weight_movement_bytes=75497472 total_nvlink_bytes=515607312 weight_to_token_ratio=0.171542 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.247928794e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53530 payload_rows_remote=49480 dup_rows=12006 dup_rows_remote=10842 dedup_ratio=0.183197 router_tokens_per_expert=[0, 0, 483, 0, 4096, 1, 4064, 4027, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 643, 0, 0, 3425, 8, 0, 414, 0, 0, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2802, 3986, 2943, 0, 1] payload_rows_per_dst=[8108, 8192, 8192, 3378, 5598, 8128, 4050, 7884] dup_rows_per_dst=[914, 2, 16, 586, 416, 0, 1164, 8908] combine_rows_per_src=[8108, 8192, 8192, 3378, 5598, 8128, 4050, 7884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62834 payload_rows_remote=54656 dup_rows=2702 dup_rows_remote=2702 dedup_ratio=0.041229 router_tokens_per_expert=[0, 0, 34, 0, 0, 0, 4069, 7, 84, 0, 0, 29, 4042, 0, 0, 691, 0, 0, 3829, 4, 4057, 4, 1, 4, 0, 0, 3607, 4013, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 5, 1, 0, 14, 0, 2, 3266, 0, 0, 2, 5, 0, 0, 2, 0, 0, 614, 0, 0, 0, 0, 52] payload_rows_per_dst=[8186, 8190, 8132, 8164, 8178, 7244, 6548, 8192] dup_rows_per_dst=[34, 1502, 8, 344, 0, 14, 2, 798] combine_rows_per_src=[8186, 8190, 8132, 8164, 8178, 7244, 6548, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57274 payload_rows_remote=49358 dup_rows=8262 dup_rows_remote=2524 dedup_ratio=0.126068 router_tokens_per_expert=[0, 0, 0, 0, 0, 9, 4086, 0, 0, 54, 13, 0, 2220, 2, 0, 7, 0, 0, 4007, 1, 2982, 1330, 0, 35, 2, 0, 3869, 4089, 1, 0, 0, 0, 0, 0, 100, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 2316, 1, 0, 31, 0, 2, 106, 4, 0, 2, 0, 0, 0, 0, 0, 447] payload_rows_per_dst=[8180, 4454, 7402, 8182, 8188, 7916, 4856, 8096] dup_rows_per_dst=[10, 138, 1294, 2, 200, 5738, 64, 816] combine_rows_per_src=[8180, 4454, 7402, 8182, 8188, 7916, 4856, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55152 payload_rows_remote=47088 dup_rows=10384 dup_rows_remote=2504 dedup_ratio=0.158447 router_tokens_per_expert=[0, 0, 0, 714, 35, 36, 4013, 0, 0, 2, 0, 0, 4026, 0, 0, 0, 4, 0, 4043, 98, 3944, 2140, 179, 168, 0, 2, 4081, 820, 21, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4080, 0, 45, 0, 0, 0, 0, 0, 1, 288, 3811, 61, 7, 54, 42, 1, 0, 3, 2, 0, 0, 41, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8064, 4932, 6396, 8160, 3378, 7874, 8168] dup_rows_per_dst=[1416, 7880, 246, 524, 0, 230, 84, 4] combine_rows_per_src=[8180, 8064, 4932, 6396, 8160, 3378, 7874, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60448 payload_rows_remote=54362 dup_rows=5088 dup_rows_remote=4096 dedup_ratio=0.077637 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 3283, 0, 0, 44, 0, 5, 3422, 0, 3320, 1182, 0, 4, 0, 0, 4096, 4079, 353, 0, 0, 0, 0, 0, 101, 0, 0, 0, 4094, 0, 22, 0, 0, 0, 0, 0, 0, 558, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6578, 6086, 8166, 8188, 8192, 8190, 6856] dup_rows_per_dst=[0, 2032, 992, 698, 202, 1160, 4, 0] combine_rows_per_src=[8192, 6578, 6086, 8166, 8188, 8192, 8190, 6856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=186441728 dispatch_meta_bytes=229560 combine_payload_bytes=186441728 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=186671288 ingress_bytes=236773376 link_bytes=236773376 token_bytes=373113016 weight_movement_bytes=50331648 total_nvlink_bytes=423444664 weight_to_token_ratio=0.134897 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.801571815e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55734 payload_rows_remote=47990 dup_rows=9802 dup_rows_remote=9468 dedup_ratio=0.149567 router_tokens_per_expert=[0, 0, 5, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 3600, 2, 5, 0, 0, 0, 3849, 3, 4084, 428, 0, 17, 0, 0, 4084, 4057, 0, 0, 0, 0, 0, 0, 372, 0, 11, 0, 4041, 0, 0, 0, 0, 0, 3, 1, 0, 0, 3908, 167, 1, 0, 13, 0, 1, 0, 23, 0, 1, 1, 0, 0, 0, 1] payload_rows_per_dst=[6346, 7210, 7858, 3618, 6592, 8174, 8192, 7744] dup_rows_per_dst=[2, 4, 1386, 0, 4, 2, 8070, 334] combine_rows_per_src=[6346, 7210, 7858, 3618, 6592, 8174, 8192, 7744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=202670080 dispatch_meta_bytes=241288 combine_payload_bytes=202670080 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=202911368 ingress_bytes=253001728 link_bytes=253001728 token_bytes=405581448 weight_movement_bytes=50331648 total_nvlink_bytes=455913096 weight_to_token_ratio=0.124098 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.925050823e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=202170368 dispatch_meta_bytes=207528 combine_payload_bytes=202170368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202377896 ingress_bytes=227336192 link_bytes=227336192 token_bytes=404548264 weight_movement_bytes=25165824 total_nvlink_bytes=429714088 weight_to_token_ratio=0.062207 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.729765749e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=223870976 dispatch_meta_bytes=229432 combine_payload_bytes=223870976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=224100408 ingress_bytes=274202624 link_bytes=274202624 token_bytes=447971384 weight_movement_bytes=50331648 total_nvlink_bytes=498303032 weight_to_token_ratio=0.112355 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=27392 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.086365145e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=192872448 dispatch_meta_bytes=198368 combine_payload_bytes=192872448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193070816 ingress_bytes=218038272 link_bytes=218038272 token_bytes=385943264 weight_movement_bytes=25165824 total_nvlink_bytes=411109088 weight_to_token_ratio=0.065206 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.659019321e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222666752 dispatch_meta_bytes=233832 combine_payload_bytes=222666752 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=222900584 ingress_bytes=323330048 link_bytes=323330048 token_bytes=445567336 weight_movement_bytes=100663296 total_nvlink_bytes=546230632 weight_to_token_ratio=0.225922 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.460168078e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60474 payload_rows_remote=55210 dup_rows=5062 dup_rows_remote=4836 dedup_ratio=0.077240 router_tokens_per_expert=[0, 0, 4009, 0, 0, 8, 4093, 0, 0, 0, 0, 71, 28, 3549, 0, 0, 3958, 476, 0, 155, 0, 0, 417, 591, 0, 0, 923, 0, 0, 0, 0, 1523, 0, 56, 0, 0, 351, 4, 0, 1, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 3873, 4096, 0, 0, 0, 1, 4093, 1, 479, 0, 0, 4, 0] payload_rows_per_dst=[7426, 7278, 8192, 5264, 8192, 8190, 7746, 8186] dup_rows_per_dst=[10, 18, 3002, 226, 824, 12, 0, 970] combine_rows_per_src=[7426, 7278, 8192, 5264, 8192, 8190, 7746, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=196567040 dispatch_meta_bytes=229832 combine_payload_bytes=196567040 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=196796872 ingress_bytes=246898688 link_bytes=246898688 token_bytes=393363912 weight_movement_bytes=50331648 total_nvlink_bytes=443695560 weight_to_token_ratio=0.127952 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.878613740e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=226140160 dispatch_meta_bytes=240184 combine_payload_bytes=226140160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=226380344 ingress_bytes=251305984 link_bytes=251305984 token_bytes=452520504 weight_movement_bytes=25165824 total_nvlink_bytes=477686328 weight_to_token_ratio=0.055613 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=14336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.912148170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48670 payload_rows_remote=40478 dup_rows=16866 dup_rows_remote=8232 dedup_ratio=0.257355 router_tokens_per_expert=[0, 7, 4050, 23, 0, 4014, 4095, 0, 1, 0, 0, 0, 1, 2328, 0, 1, 4061, 0, 9, 0, 1, 0, 50, 0, 0, 26, 0, 0, 0, 7, 0, 1205, 0, 0, 0, 0, 3661, 1, 16, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3314, 2098, 0, 0, 0, 2, 3747, 31, 13, 0, 0, 3, 1] payload_rows_per_dst=[8192, 8158, 6716, 2466, 8000, 8192, 0, 6946] dup_rows_per_dst=[8634, 4626, 32, 10, 3552, 2, 0, 10] combine_rows_per_src=[8192, 8158, 6716, 2466, 8000, 8192, 0, 6946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49496 payload_rows_remote=41526 dup_rows=16040 dup_rows_remote=16014 dedup_ratio=0.244751 router_tokens_per_expert=[0, 0, 3890, 638, 0, 133, 3940, 0, 0, 0, 0, 0, 0, 2403, 0, 0, 4094, 3, 1, 0, 0, 0, 1879, 271, 221, 284, 3573, 0, 0, 0, 1, 0, 0, 0, 35, 0, 1786, 0, 0, 405, 0, 13, 0, 0, 0, 0, 0, 45, 0, 0, 0, 1876, 3597, 0, 0, 0, 0, 3680, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1532, 4806, 8192, 8192, 7642, 7970, 3802, 7360] dup_rows_per_dst=[10, 0, 4304, 7746, 3622, 26, 332, 0] combine_rows_per_src=[1532, 4806, 8192, 8192, 7642, 7970, 3802, 7360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57870 payload_rows_remote=49698 dup_rows=7666 dup_rows_remote=6496 dedup_ratio=0.116974 router_tokens_per_expert=[0, 0, 4063, 0, 0, 427, 4078, 0, 46, 2, 3, 0, 0, 4078, 0, 1, 4044, 0, 105, 131, 0, 0, 0, 5, 1, 0, 1895, 0, 0, 1, 0, 180, 0, 0, 0, 2, 581, 0, 0, 3, 0, 0, 0, 0, 3, 1, 1, 29, 3, 0, 0, 3517, 4085, 0, 1106, 0, 1, 4091, 243, 19, 7, 0, 16, 0] payload_rows_per_dst=[854, 8192, 8188, 8184, 8172, 8190, 7898, 8192] dup_rows_per_dst=[0, 68, 382, 4096, 1170, 34, 1354, 562] combine_rows_per_src=[854, 8192, 8188, 8184, 8172, 8190, 7898, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51180 payload_rows_remote=44710 dup_rows=14356 dup_rows_remote=12366 dedup_ratio=0.219055 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4038, 11, 3910, 2958, 1, 2, 0, 0, 4086, 3101, 0, 0, 0, 0, 0, 0, 4, 0, 24, 0, 4063, 1, 0, 0, 0, 0, 0, 0, 0, 3373, 3079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 20, 8188, 6202, 5824, 8190, 6470, 8094] dup_rows_per_dst=[0, 0, 5576, 0, 58, 6728, 1990, 4] combine_rows_per_src=[8192, 20, 8188, 6202, 5824, 8190, 6470, 8094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51350 payload_rows_remote=43198 dup_rows=14186 dup_rows_remote=6302 dedup_ratio=0.216461 router_tokens_per_expert=[0, 0, 3761, 1165, 2, 9, 4094, 7, 0, 0, 0, 0, 0, 3946, 0, 0, 4072, 0, 0, 2, 0, 0, 622, 0, 10, 0, 2830, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1902, 1325, 63, 0, 0, 9, 0, 19, 0, 140, 12, 0, 0, 2, 0, 2408, 1914, 0, 0, 0, 531, 3902, 0, 0, 0, 0, 0, 4] payload_rows_per_dst=[7972, 8152, 4292, 5680, 7512, 8192, 1358, 8192] dup_rows_per_dst=[1916, 7884, 418, 0, 2930, 356, 0, 682] combine_rows_per_src=[7972, 8152, 4292, 5680, 7512, 8192, 1358, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59900 payload_rows_remote=55198 dup_rows=5636 dup_rows_remote=5438 dedup_ratio=0.085999 router_tokens_per_expert=[0, 0, 4095, 960, 0, 23, 4006, 0, 5, 0, 0, 1, 0, 3056, 0, 0, 4093, 0, 1, 2, 0, 0, 727, 107, 0, 0, 2675, 0, 0, 0, 0, 794, 0, 0, 0, 0, 146, 18, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 3867, 4094, 0, 0, 0, 26, 3928, 3, 0, 0, 0, 1, 0] payload_rows_per_dst=[8190, 8146, 4702, 6868, 8190, 8012, 7936, 7856] dup_rows_per_dst=[1966, 2938, 198, 70, 330, 0, 74, 60] combine_rows_per_src=[8190, 8146, 4702, 6868, 8190, 8012, 7936, 7856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=165797888 dispatch_meta_bytes=194840 combine_payload_bytes=165797888 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=165992728 ingress_bytes=291627008 link_bytes=291627008 token_bytes=331790616 weight_movement_bytes=125829120 total_nvlink_bytes=457619736 weight_to_token_ratio=0.379243 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.218944574e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51146 payload_rows_remote=42960 dup_rows=14390 dup_rows_remote=14194 dedup_ratio=0.219574 router_tokens_per_expert=[0, 0, 3915, 944, 10, 4055, 4040, 0, 0, 3, 1, 0, 0, 806, 0, 0, 4092, 0, 12, 1, 0, 0, 259, 0, 0, 6, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 0, 125, 0, 0, 0, 0, 18, 7, 0, 0, 3, 6, 0, 0, 3475, 4088, 0, 11, 0, 0, 4083, 0, 2, 0, 0, 106, 0] payload_rows_per_dst=[8168, 1618, 8184, 5400, 3286, 8116, 8188, 8186] dup_rows_per_dst=[1850, 2, 544, 1212, 0, 56, 10530, 196] combine_rows_per_src=[8168, 1618, 8184, 5400, 3286, 8116, 8188, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57104 payload_rows_remote=48974 dup_rows=8432 dup_rows_remote=5482 dedup_ratio=0.128662 router_tokens_per_expert=[0, 0, 0, 0, 2, 475, 0, 0, 0, 3, 0, 0, 9, 41, 27, 3, 0, 0, 0, 0, 56, 0, 3791, 67, 0, 4038, 0, 0, 0, 0, 336, 0, 0, 0, 7, 3781, 0, 0, 1, 1, 4053, 0, 0, 0, 4078, 4059, 2945, 0, 157, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 1, 801, 3925] payload_rows_per_dst=[8160, 8126, 7764, 8130, 7580, 7404, 8114, 1826] dup_rows_per_dst=[950, 158, 64, 2950, 0, 4260, 50, 0] combine_rows_per_src=[8160, 8126, 7764, 8130, 7580, 7404, 8114, 1826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=170090496 dispatch_meta_bytes=230160 combine_payload_bytes=170090496 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=170320656 ingress_bytes=220422144 link_bytes=220422144 token_bytes=340411152 weight_movement_bytes=50331648 total_nvlink_bytes=390742800 weight_to_token_ratio=0.147855 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.677157832e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=203563008 dispatch_meta_bytes=224776 combine_payload_bytes=203563008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203787784 ingress_bytes=228728832 link_bytes=228728832 token_bytes=407350792 weight_movement_bytes=25165824 total_nvlink_bytes=432516616 weight_to_token_ratio=0.061779 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.740362130e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=183132160 dispatch_meta_bytes=228304 combine_payload_bytes=183132160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183360464 ingress_bytes=208297984 link_bytes=208297984 token_bytes=366492624 weight_movement_bytes=25165824 total_nvlink_bytes=391658448 weight_to_token_ratio=0.068667 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=27136 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.584906983e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=176939008 dispatch_meta_bytes=198000 combine_payload_bytes=176939008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=177137008 ingress_bytes=202104832 link_bytes=202104832 token_bytes=354076016 weight_movement_bytes=25165824 total_nvlink_bytes=379241840 weight_to_token_ratio=0.071075 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.537784252e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=226091008 dispatch_meta_bytes=242544 combine_payload_bytes=226091008 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=226333552 ingress_bytes=276422656 link_bytes=276422656 token_bytes=452424560 weight_movement_bytes=50331648 total_nvlink_bytes=502756208 weight_to_token_ratio=0.111249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.103257023e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=175964160 dispatch_meta_bytes=228616 combine_payload_bytes=175964160 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=176192776 ingress_bytes=201129984 link_bytes=201129984 token_bytes=352156936 weight_movement_bytes=25165824 total_nvlink_bytes=377322760 weight_to_token_ratio=0.071462 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.530366785e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=200597504 dispatch_meta_bytes=217824 combine_payload_bytes=200597504 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200815328 ingress_bytes=225763328 link_bytes=225763328 token_bytes=401412832 weight_movement_bytes=25165824 total_nvlink_bytes=426578656 weight_to_token_ratio=0.062693 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.717798071e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50182 payload_rows_remote=42594 dup_rows=15354 dup_rows_remote=15324 dedup_ratio=0.234283 router_tokens_per_expert=[0, 6, 4, 0, 0, 370, 0, 0, 0, 0, 0, 1, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 26, 0, 0, 4, 0, 996, 3727, 0, 3380, 4, 3218, 0, 0, 0, 2, 4089, 0, 92, 0, 3429, 4095, 120, 0, 9, 0, 0, 0, 0, 0, 250, 0, 0, 2, 0, 0, 0, 0, 796, 4044] payload_rows_per_dst=[7588, 8192, 8186, 8192, 8150, 184, 8094, 1596] dup_rows_per_dst=[30, 262, 1154, 9492, 3904, 0, 512, 0] combine_rows_per_src=[7588, 8192, 8186, 8192, 8150, 184, 8094, 1596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52790 payload_rows_remote=44642 dup_rows=12746 dup_rows_remote=5412 dedup_ratio=0.194489 router_tokens_per_expert=[0, 0, 268, 1, 0, 0, 93, 0, 0, 0, 0, 0, 199, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3926, 0, 0, 77, 0, 3, 7, 4, 236, 1500, 0, 460, 102, 525, 0, 0, 0, 0, 3940, 0, 0, 0, 3596, 3927, 3801, 0, 27, 0, 1, 0, 0, 0, 1877, 1, 0, 0, 0, 0, 1, 15, 4079, 4096] payload_rows_per_dst=[7604, 8184, 7856, 3624, 994, 8148, 8192, 8188] dup_rows_per_dst=[312, 76, 0, 30, 130, 7334, 3812, 1052] combine_rows_per_src=[7604, 8184, 7856, 3624, 994, 8148, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56434 payload_rows_remote=48242 dup_rows=9102 dup_rows_remote=8970 dedup_ratio=0.138885 router_tokens_per_expert=[1, 22, 111, 0, 8, 9, 1, 0, 0, 0, 6, 0, 0, 2, 0, 0, 13, 0, 0, 0, 24, 0, 3944, 1, 0, 2, 0, 5, 40, 0, 22, 566, 1, 4005, 0, 3254, 52, 0, 0, 936, 3178, 0, 58, 0, 4078, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 5, 0, 0, 4067, 4062] payload_rows_per_dst=[8188, 8192, 7952, 7528, 8190, 116, 8124, 8144] dup_rows_per_dst=[272, 132, 12, 98, 8306, 0, 282, 0] combine_rows_per_src=[8188, 8192, 7952, 7528, 8190, 116, 8124, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54148 payload_rows_remote=48526 dup_rows=11388 dup_rows_remote=11384 dedup_ratio=0.173767 router_tokens_per_expert=[1, 13, 0, 0, 11, 1297, 0, 0, 0, 2, 0, 0, 0, 8, 14, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 6, 4010, 2, 4, 0, 0, 305, 30, 2, 3, 0, 4095, 0, 0, 0, 0, 4053, 0, 0, 0, 3475, 3990, 2200, 0, 229, 0, 0, 1, 0, 0, 2860, 0, 24, 0, 0, 0, 1, 0, 54, 2003] payload_rows_per_dst=[8012, 8028, 8150, 8032, 5622, 8152, 7606, 546] dup_rows_per_dst=[1582, 0, 0, 682, 4, 6530, 2580, 10] combine_rows_per_src=[8012, 8028, 8150, 8032, 5622, 8152, 7606, 546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=174465024 dispatch_meta_bytes=231672 combine_payload_bytes=174465024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=174696696 ingress_bytes=199630848 link_bytes=199630848 token_bytes=349161720 weight_movement_bytes=25165824 total_nvlink_bytes=374327544 weight_to_token_ratio=0.072075 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.518960093e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54778 payload_rows_remote=46586 dup_rows=10758 dup_rows_remote=2632 dedup_ratio=0.164154 router_tokens_per_expert=[0, 2, 3943, 371, 0, 4035, 4094, 0, 1, 0, 0, 0, 0, 1824, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 987, 0, 0, 1, 0, 130, 0, 0, 0, 0, 1031, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3942, 0, 121, 0, 0, 3965, 0, 102, 0, 0, 2, 0] payload_rows_per_dst=[8192, 3650, 8190, 8178, 2114, 8188, 8192, 8074] dup_rows_per_dst=[624, 0, 0, 1944, 0, 0, 8126, 64] combine_rows_per_src=[8192, 3650, 8190, 8178, 2114, 8188, 8192, 8074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59092 payload_rows_remote=51028 dup_rows=6444 dup_rows_remote=6444 dedup_ratio=0.098328 router_tokens_per_expert=[0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 38, 0, 0, 0, 0, 596, 133, 0, 2188, 1, 4096, 0, 0, 0, 0, 3384, 0, 96, 0, 4096, 3754, 1739, 0, 607, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 3897, 4086] payload_rows_per_dst=[8192, 7510, 8064, 7782, 8192, 3372, 8186, 7794] dup_rows_per_dst=[24, 296, 0, 520, 4378, 0, 1226, 0] combine_rows_per_src=[8192, 7510, 8064, 7782, 8192, 3372, 8186, 7794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47366 payload_rows_remote=39178 dup_rows=18170 dup_rows_remote=3172 dedup_ratio=0.277252 router_tokens_per_expert=[0, 5, 0, 0, 11, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 3, 0, 0, 684, 579, 0, 4088, 2, 4074, 0, 0, 0, 43, 681, 0, 59, 0, 4045, 4094, 2669, 0, 4, 0, 83, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3530, 3989] payload_rows_per_dst=[4942, 8188, 8140, 2482, 8178, 7076, 172, 8188] dup_rows_per_dst=[12, 0, 0, 50, 88, 2938, 84, 14998] combine_rows_per_src=[4942, 8188, 8140, 2482, 8178, 7076, 172, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55456 payload_rows_remote=47284 dup_rows=10080 dup_rows_remote=10080 dedup_ratio=0.153809 router_tokens_per_expert=[0, 4086, 3304, 1, 6, 0, 0, 0, 0, 4096, 0, 831, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 605, 0, 1, 0, 0, 2, 0, 3892, 0, 0, 0, 0, 0, 688, 3694, 0, 0, 3346, 0, 0, 0, 1, 2, 4, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[6612, 8192, 8192, 8172, 1210, 8188, 6698, 8192] dup_rows_per_dst=[10, 1662, 34, 0, 6, 8360, 0, 8] combine_rows_per_src=[6612, 8192, 8192, 8172, 1210, 8188, 6698, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=182853632 dispatch_meta_bytes=200216 combine_payload_bytes=182853632 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=183053848 ingress_bytes=308682752 link_bytes=308682752 token_bytes=365907480 weight_movement_bytes=125829120 total_nvlink_bytes=491736600 weight_to_token_ratio=0.343882 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=14976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.348719080e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=198762496 dispatch_meta_bytes=239640 combine_payload_bytes=198762496 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=199002136 ingress_bytes=274259968 link_bytes=274259968 token_bytes=397764632 weight_movement_bytes=75497472 total_nvlink_bytes=473262104 weight_to_token_ratio=0.189804 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.086801467e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=197599232 dispatch_meta_bytes=228848 combine_payload_bytes=197599232 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=197828080 ingress_bytes=247930880 link_bytes=247930880 token_bytes=395427312 weight_movement_bytes=50331648 total_nvlink_bytes=445758960 weight_to_token_ratio=0.127284 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.886467528e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=190816256 dispatch_meta_bytes=196872 combine_payload_bytes=190816256 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=191013128 ingress_bytes=266313728 link_bytes=266313728 token_bytes=381829384 weight_movement_bytes=75497472 total_nvlink_bytes=457326856 weight_to_token_ratio=0.197726 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21504 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.026339762e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=209010688 dispatch_meta_bytes=229888 combine_payload_bytes=209010688 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=209240576 ingress_bytes=234176512 link_bytes=234176512 token_bytes=418251264 weight_movement_bytes=25165824 total_nvlink_bytes=443417088 weight_to_token_ratio=0.060169 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781812680e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160473088 dispatch_meta_bytes=169400 combine_payload_bytes=160473088 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160642488 ingress_bytes=210804736 link_bytes=210804736 token_bytes=321115576 weight_movement_bytes=50331648 total_nvlink_bytes=371447224 weight_to_token_ratio=0.156740 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.603980470e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=193675264 dispatch_meta_bytes=229456 combine_payload_bytes=193675264 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=193904720 ingress_bytes=244006912 link_bytes=244006912 token_bytes=387579984 weight_movement_bytes=50331648 total_nvlink_bytes=437911632 weight_to_token_ratio=0.129861 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.856610666e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53678 payload_rows_remote=45486 dup_rows=11858 dup_rows_remote=8858 dedup_ratio=0.180939 router_tokens_per_expert=[0, 3999, 4096, 144, 1355, 1, 0, 0, 0, 4095, 0, 367, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 17, 0, 4094, 893, 0, 0, 0, 0, 0, 0, 4051, 0, 1, 2, 0, 0, 0, 3375, 0, 0, 2434, 0, 0, 0, 0, 0, 0, 0, 1, 0, 70, 3764, 0, 0] payload_rows_per_dst=[8192, 8190, 8188, 8032, 1786, 6754, 4868, 7668] dup_rows_per_dst=[3000, 8844, 0, 10, 0, 2, 0, 2] combine_rows_per_src=[8192, 8190, 8188, 8032, 1786, 6754, 4868, 7668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=186310656 dispatch_meta_bytes=217376 combine_payload_bytes=186310656 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=186528032 ingress_bytes=286973952 link_bytes=286973952 token_bytes=372838688 weight_movement_bytes=100663296 total_nvlink_bytes=473501984 weight_to_token_ratio=0.269992 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.183540195e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52674 payload_rows_remote=52578 dup_rows=12862 dup_rows_remote=12862 dedup_ratio=0.196259 router_tokens_per_expert=[0, 3990, 4081, 0, 3951, 0, 0, 60, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 103, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4077, 1, 0, 21, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 2, 38, 0, 0, 93, 0, 2, 0, 1, 1, 0, 8, 0, 1, 0, 4038, 0, 0] payload_rows_per_dst=[8192, 3742, 8192, 7982, 8188, 96, 8192, 8090] dup_rows_per_dst=[7992, 0, 216, 0, 4456, 0, 194, 4] combine_rows_per_src=[8192, 3742, 8192, 7982, 8188, 96, 8192, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59500 payload_rows_remote=51308 dup_rows=6036 dup_rows_remote=5736 dedup_ratio=0.092102 router_tokens_per_expert=[0, 4067, 3319, 398, 0, 2, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 43, 0, 10, 1, 0, 0, 0, 0, 0, 0, 4041, 4068, 0, 0, 0, 138, 40, 0, 3881, 0, 0, 0, 1228, 0, 8, 0, 0, 0, 4017, 0, 24, 0, 0, 0, 3, 0, 0, 2, 0, 3103, 0, 277] payload_rows_per_dst=[7420, 8162, 8120, 8156, 8192, 4616, 8072, 6762] dup_rows_per_dst=[18, 0, 82, 0, 300, 2042, 3586, 8] combine_rows_per_src=[7420, 8162, 8120, 8156, 8192, 4616, 8072, 6762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55012 payload_rows_remote=46820 dup_rows=10524 dup_rows_remote=608 dedup_ratio=0.160583 router_tokens_per_expert=[1, 3307, 4087, 0, 0, 9, 0, 2, 0, 4032, 9, 3258, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4054, 3494, 13, 227, 0, 0, 0, 0, 3765, 2, 0, 0, 0, 0, 0, 854, 30, 0, 1523, 5, 0, 0, 0, 0, 0, 0, 85, 0, 0, 3901, 48, 0] payload_rows_per_dst=[8180, 8192, 8134, 6688, 7016, 5728, 3106, 7968] dup_rows_per_dst=[18, 9916, 24, 0, 452, 4, 10, 100] combine_rows_per_src=[8180, 8192, 8134, 6688, 7016, 5728, 3106, 7968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58514 payload_rows_remote=50322 dup_rows=7022 dup_rows_remote=5582 dedup_ratio=0.107147 router_tokens_per_expert=[0, 4095, 4075, 33, 1099, 0, 0, 4, 0, 4096, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 592, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 231, 1354, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8192, 1184, 8192, 8184, 8192] dup_rows_per_dst=[2236, 168, 1440, 4, 0, 3174, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8192, 1184, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53672 payload_rows_remote=46066 dup_rows=11864 dup_rows_remote=10968 dedup_ratio=0.181030 router_tokens_per_expert=[0, 0, 11, 0, 931, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 249, 0, 0, 2, 0, 570, 2151, 0, 910, 0, 2263, 0, 0, 0, 0, 3309, 0, 0, 0, 3882, 4096, 1702, 0, 679, 0, 33, 0, 0, 0, 3096, 0, 0, 0, 0, 0, 0, 0, 681, 4096] payload_rows_per_dst=[7784, 8192, 8192, 5006, 1820, 7130, 7606, 7942] dup_rows_per_dst=[1884, 0, 2, 938, 0, 2892, 896, 5252] combine_rows_per_src=[7784, 8192, 8192, 5006, 1820, 7130, 7606, 7942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58382 payload_rows_remote=50190 dup_rows=7154 dup_rows_remote=6890 dedup_ratio=0.109161 router_tokens_per_expert=[2314, 0, 0, 0, 4003, 88, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 118, 0, 0, 0, 0, 34, 3104, 0, 0, 0, 139, 0, 0, 0, 4089, 0, 863, 0, 1434, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1168, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3152, 0, 0, 4039] payload_rows_per_dst=[4804, 8192, 6406, 8192, 8124, 8192, 7812, 6660] dup_rows_per_dst=[0, 6, 106, 264, 4476, 40, 1506, 756] combine_rows_per_src=[4804, 8192, 6406, 8192, 8124, 8192, 7812, 6660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47240 payload_rows_remote=39052 dup_rows=18296 dup_rows_remote=16136 dedup_ratio=0.279175 router_tokens_per_expert=[30, 4082, 4094, 31, 539, 0, 0, 10, 0, 4094, 0, 105, 2, 0, 0, 0, 2, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4037, 3257, 0, 12, 0, 46, 0, 0, 4086, 0, 0, 0, 13, 0, 0, 2409, 5, 0, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4088, 0, 6] payload_rows_per_dst=[1120, 214, 6516, 8188, 8192, 6638, 8184, 8188] dup_rows_per_dst=[100, 0, 2, 1414, 8192, 2822, 3606, 2160] combine_rows_per_src=[1120, 214, 6516, 8188, 8192, 6638, 8184, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=215359488 dispatch_meta_bytes=261760 combine_payload_bytes=215359488 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=215621248 ingress_bytes=290856960 link_bytes=290856960 token_bytes=430980736 weight_movement_bytes=75497472 total_nvlink_bytes=506478208 weight_to_token_ratio=0.175176 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=14720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.213085399e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=210157568 dispatch_meta_bytes=228176 combine_payload_bytes=210157568 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=210385744 ingress_bytes=260489216 link_bytes=260489216 token_bytes=420543312 weight_movement_bytes=50331648 total_nvlink_bytes=470874960 weight_to_token_ratio=0.119682 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.982021955e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=191774720 dispatch_meta_bytes=189712 combine_payload_bytes=191774720 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=191964432 ingress_bytes=242106368 link_bytes=242106368 token_bytes=383739152 weight_movement_bytes=50331648 total_nvlink_bytes=434070800 weight_to_token_ratio=0.131161 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.842149722e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=206118912 dispatch_meta_bytes=223616 combine_payload_bytes=206118912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206342528 ingress_bytes=231284736 link_bytes=231284736 token_bytes=412461440 weight_movement_bytes=25165824 total_nvlink_bytes=437627264 weight_to_token_ratio=0.061014 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.759809606e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=188686336 dispatch_meta_bytes=228136 combine_payload_bytes=188686336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188914472 ingress_bytes=213852160 link_bytes=213852160 token_bytes=377600808 weight_movement_bytes=25165824 total_nvlink_bytes=402766632 weight_to_token_ratio=0.066647 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.627167845e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=205578240 dispatch_meta_bytes=228320 combine_payload_bytes=205578240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=205806560 ingress_bytes=230744064 link_bytes=230744064 token_bytes=411384800 weight_movement_bytes=25165824 total_nvlink_bytes=436550624 weight_to_token_ratio=0.061173 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.755695717e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50106 payload_rows_remote=41914 dup_rows=15430 dup_rows_remote=72 dedup_ratio=0.235443 router_tokens_per_expert=[4095, 6, 3, 0, 3963, 0, 0, 0, 1, 0, 0, 3, 5, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4091, 0, 0, 5, 0, 0, 0, 1, 4076, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4068, 0, 0, 0, 0, 4093, 0, 0, 0, 8, 0, 0, 134, 0, 2, 0, 0, 0, 4095] payload_rows_per_dst=[8192, 8186, 8192, 8162, 780, 8136, 8186, 272] dup_rows_per_dst=[15358, 4, 50, 2, 0, 0, 16, 0] combine_rows_per_src=[8192, 8186, 8192, 8162, 780, 8136, 8186, 272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159956992 dispatch_meta_bytes=220752 combine_payload_bytes=159956992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160177744 ingress_bytes=185122816 link_bytes=185122816 token_bytes=320134736 weight_movement_bytes=25165824 total_nvlink_bytes=345300560 weight_to_token_ratio=0.078610 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408570732e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=171679744 dispatch_meta_bytes=167944 combine_payload_bytes=171679744 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=171847688 ingress_bytes=222011392 link_bytes=222011392 token_bytes=343527432 weight_movement_bytes=50331648 total_nvlink_bytes=393859080 weight_to_token_ratio=0.146514 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=23424 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.689250173e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58722 payload_rows_remote=53244 dup_rows=6814 dup_rows_remote=6804 dedup_ratio=0.103973 router_tokens_per_expert=[4066, 0, 1, 0, 2867, 0, 0, 35, 0, 2, 0, 0, 11, 12, 0, 0, 865, 1, 0, 0, 0, 1041, 4096, 0, 0, 0, 0, 0, 1, 0, 3919, 2, 251, 0, 0, 39, 0, 0, 0, 0, 9, 0, 3938, 0, 8, 2727, 0, 0, 0, 0, 3026, 0, 0, 0, 558, 0, 11, 882, 4, 0, 312, 0, 0, 4084] payload_rows_per_dst=[8164, 7914, 8192, 7844, 6254, 5478, 6684, 8192] dup_rows_per_dst=[40, 12, 3814, 0, 60, 10, 484, 2394] combine_rows_per_src=[8164, 7914, 8192, 7844, 6254, 5478, 6684, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59888 payload_rows_remote=51710 dup_rows=5648 dup_rows_remote=3934 dedup_ratio=0.086182 router_tokens_per_expert=[1870, 0, 41, 0, 4089, 0, 0, 9, 1, 68, 0, 2, 179, 295, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 0, 857, 0, 0, 0, 0, 0, 1176, 0, 3910, 0, 1, 3601, 0, 0, 0, 0, 4068, 0, 3, 0, 0, 2, 27, 185, 0, 0, 525, 0, 0, 4060] payload_rows_per_dst=[3822, 8192, 8186, 7410, 8178, 7762, 8146, 8192] dup_rows_per_dst=[18, 718, 2, 0, 1714, 1794, 0, 1402] combine_rows_per_src=[3822, 8192, 8186, 7410, 8178, 7762, 8146, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60336 payload_rows_remote=54836 dup_rows=5200 dup_rows_remote=5028 dedup_ratio=0.079346 router_tokens_per_expert=[4083, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 28, 0, 0, 0, 0, 580, 2228, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 54, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4057, 0, 0, 4094, 0, 0, 0, 0, 2428, 6, 0, 5, 0, 0, 0, 0, 0, 0, 2881, 0, 0, 4096] payload_rows_per_dst=[8166, 8160, 5500, 8192, 8176, 8188, 8192, 5762] dup_rows_per_dst=[0, 4, 172, 28, 108, 10, 4878, 0] combine_rows_per_src=[8166, 8160, 5500, 8192, 8176, 8188, 8192, 5762] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54490 payload_rows_remote=46310 dup_rows=11046 dup_rows_remote=10800 dedup_ratio=0.168549 router_tokens_per_expert=[1180, 0, 0, 0, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 122, 0, 0, 0, 305, 4092, 0, 0, 30, 0, 0, 19, 8, 4022, 0, 3552, 2, 0, 0, 2, 0, 13, 0, 15, 0, 3961, 464, 51, 2499, 0, 0, 0, 0, 926, 0, 0, 3, 443, 1, 63, 0, 0, 9, 3302, 0, 3, 3787] payload_rows_per_dst=[4498, 8180, 8190, 8120, 7924, 5926, 4980, 6672] dup_rows_per_dst=[1056, 246, 1512, 648, 6498, 132, 872, 82] combine_rows_per_src=[4498, 8180, 8190, 8120, 7924, 5926, 4980, 6672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51306 payload_rows_remote=43116 dup_rows=14230 dup_rows_remote=14216 dedup_ratio=0.217133 router_tokens_per_expert=[616, 4096, 2, 3416, 0, 0, 0, 0, 4096, 0, 0, 2, 28, 0, 3666, 7, 0, 0, 0, 0, 0, 0, 610, 352, 0, 0, 0, 9, 0, 2, 4091, 0, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 960, 4096, 0, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291] payload_rows_per_dst=[1236, 8192, 3520, 8190, 8192, 6412, 8150, 7414] dup_rows_per_dst=[0, 7406, 184, 14, 3642, 1920, 1064, 0] combine_rows_per_src=[1236, 8192, 3520, 8190, 8192, 6412, 8150, 7414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51916 payload_rows_remote=43724 dup_rows=13620 dup_rows_remote=10976 dedup_ratio=0.207825 router_tokens_per_expert=[2, 4051, 2065, 0, 361, 0, 0, 0, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4084, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1568, 2892, 0, 0, 1091, 0, 0, 0, 0, 231, 0, 1, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[4152, 62, 8190, 8102, 8192, 6834, 8192, 8192] dup_rows_per_dst=[108, 0, 14, 596, 8170, 2086, 2644, 2] combine_rows_per_src=[4152, 62, 8190, 8102, 8192, 6834, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54004 payload_rows_remote=45814 dup_rows=11532 dup_rows_remote=4722 dedup_ratio=0.175964 router_tokens_per_expert=[1957, 4, 0, 0, 3462, 0, 0, 5, 0, 0, 4, 0, 0, 0, 8, 0, 1120, 0, 0, 0, 0, 0, 3485, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 199, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4087, 3, 0, 4039, 0, 0, 0, 0, 2712, 0, 1, 24, 34, 0, 0, 605, 2, 0, 2793, 6, 0, 4094] payload_rows_per_dst=[3920, 7522, 5310, 8182, 7284, 8084, 5512, 8190] dup_rows_per_dst=[12, 10, 1130, 0, 38, 3502, 30, 6810] combine_rows_per_src=[3920, 7522, 5310, 8182, 7284, 8084, 5512, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=218087424 dispatch_meta_bytes=240192 combine_payload_bytes=218087424 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=218327616 ingress_bytes=268419072 link_bytes=268419072 token_bytes=436415040 weight_movement_bytes=50331648 total_nvlink_bytes=486746688 weight_to_token_ratio=0.115330 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.042358997e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211804160 dispatch_meta_bytes=222576 combine_payload_bytes=211804160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=212026736 ingress_bytes=236969984 link_bytes=236969984 token_bytes=423830896 weight_movement_bytes=25165824 total_nvlink_bytes=448996720 weight_to_token_ratio=0.059377 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25600 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.803067775e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50084 payload_rows_remote=41906 dup_rows=15452 dup_rows_remote=7370 dedup_ratio=0.235779 router_tokens_per_expert=[4089, 3779, 0, 4074, 0, 0, 0, 0, 4041, 1, 117, 0, 0, 1, 0, 0, 0, 14, 0, 0, 3, 0, 465, 4069, 0, 0, 23, 33, 5, 0, 25, 2100, 0, 0, 0, 4, 0, 0, 1, 0, 1, 0, 0, 0, 4092, 2376, 3448, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[8178, 2416, 8192, 4306, 7568, 4762, 6512, 8150] dup_rows_per_dst=[8082, 8, 5620, 66, 0, 1676, 0, 0] combine_rows_per_src=[8178, 2416, 8192, 4306, 7568, 4762, 6512, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=224608256 dispatch_meta_bytes=239456 combine_payload_bytes=224608256 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=224847712 ingress_bytes=274939904 link_bytes=274939904 token_bytes=449455968 weight_movement_bytes=50331648 total_nvlink_bytes=499787616 weight_to_token_ratio=0.111983 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=27904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.091974994e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=176603136 dispatch_meta_bytes=229328 combine_payload_bytes=176603136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=176832464 ingress_bytes=201768960 link_bytes=201768960 token_bytes=353435600 weight_movement_bytes=25165824 total_nvlink_bytes=378601424 weight_to_token_ratio=0.071203 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=13952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.535228654e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=189685760 dispatch_meta_bytes=228440 combine_payload_bytes=189685760 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189914200 ingress_bytes=214851584 link_bytes=214851584 token_bytes=379599960 weight_movement_bytes=25165824 total_nvlink_bytes=404765784 weight_to_token_ratio=0.066296 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.634772307e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=179093504 dispatch_meta_bytes=218800 combine_payload_bytes=179093504 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=179312304 ingress_bytes=204259328 link_bytes=204259328 token_bytes=358405808 weight_movement_bytes=25165824 total_nvlink_bytes=383571632 weight_to_token_ratio=0.070216 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.554177477e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=187654144 dispatch_meta_bytes=202144 combine_payload_bytes=187654144 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=187856288 ingress_bytes=288317440 link_bytes=288317440 token_bytes=375510432 weight_movement_bytes=100663296 total_nvlink_bytes=476173728 weight_to_token_ratio=0.268071 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.193762586e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=171646976 dispatch_meta_bytes=197104 combine_payload_bytes=171646976 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=171844080 ingress_bytes=297476096 link_bytes=297476096 token_bytes=343491056 weight_movement_bytes=125829120 total_nvlink_bytes=469320176 weight_to_token_ratio=0.366324 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.263449376e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64018 payload_rows_remote=55846 dup_rows=1518 dup_rows_remote=1098 dedup_ratio=0.023163 router_tokens_per_expert=[3634, 3868, 0, 4002, 0, 0, 0, 0, 4093, 1, 37, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 44, 0, 0, 19, 0, 4084, 0, 0, 34, 0, 0, 0, 0, 7, 59, 0, 0, 0, 4069, 203, 4034, 0, 0, 0, 60, 408, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 2, 6] payload_rows_per_dst=[7268, 8188, 8186, 8190, 7804, 8172, 8192, 8018] dup_rows_per_dst=[0, 78, 2, 104, 14, 420, 882, 18] combine_rows_per_src=[7268, 8188, 8186, 8190, 7804, 8172, 8192, 8018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59202 payload_rows_remote=51210 dup_rows=6334 dup_rows_remote=6334 dedup_ratio=0.096649 router_tokens_per_expert=[4096, 3987, 0, 4047, 12, 0, 0, 0, 4096, 0, 0, 93, 0, 39, 878, 0, 0, 0, 0, 0, 0, 0, 829, 524, 0, 0, 0, 208, 0, 2508, 3430, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 33, 2, 0, 4048, 1, 3866, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[8192, 8192, 2686, 8054, 7992, 7798, 8102, 8186] dup_rows_per_dst=[24, 2020, 20, 4238, 0, 6, 0, 26] combine_rows_per_src=[8192, 8192, 2686, 8054, 7992, 7798, 8102, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58066 payload_rows_remote=49874 dup_rows=7470 dup_rows_remote=7470 dedup_ratio=0.113983 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 346, 267, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 4076, 112, 97, 0, 5, 0, 0, 0, 0, 3842, 0, 3297, 0, 0, 4093] payload_rows_per_dst=[8192, 1246, 8188, 8192, 8186, 8190, 8188, 7684] dup_rows_per_dst=[0, 0, 0, 0, 2, 482, 6986, 0] combine_rows_per_src=[8192, 1246, 8188, 8192, 8186, 8190, 8188, 7684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62672 payload_rows_remote=54840 dup_rows=2864 dup_rows_remote=1072 dedup_ratio=0.043701 router_tokens_per_expert=[4096, 4026, 457, 4096, 0, 0, 0, 0, 4078, 0, 0, 21, 0, 0, 70, 0, 0, 19, 0, 0, 4, 0, 0, 1329, 0, 0, 0, 4, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3649, 3460, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7832, 6722, 8052, 7298, 8192, 8192] dup_rows_per_dst=[914, 146, 1792, 0, 0, 0, 12, 0] combine_rows_per_src=[8192, 8192, 7832, 6722, 8052, 7298, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57280 payload_rows_remote=49088 dup_rows=8256 dup_rows_remote=7848 dedup_ratio=0.125977 router_tokens_per_expert=[3994, 3800, 0, 3677, 14, 0, 0, 0, 4080, 0, 333, 0, 3, 31, 83, 1, 0, 530, 0, 0, 2, 178, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 603, 0, 17, 0, 159, 0, 0, 0, 0, 0, 4, 0, 1, 4018, 4050, 3022, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 24, 0, 0, 48, 0, 1] payload_rows_per_dst=[8170, 8192, 8192, 1206, 7916, 8110, 8042, 7452] dup_rows_per_dst=[308, 408, 7456, 0, 36, 0, 0, 48] combine_rows_per_src=[8170, 8192, 8192, 1206, 7916, 8110, 8042, 7452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=228745216 dispatch_meta_bytes=227776 combine_payload_bytes=228745216 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=228972992 ingress_bytes=304242688 link_bytes=304242688 token_bytes=457718208 weight_movement_bytes=75497472 total_nvlink_bytes=533215680 weight_to_token_ratio=0.164943 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.314935323e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57560 payload_rows_remote=49368 dup_rows=7976 dup_rows_remote=6326 dedup_ratio=0.121704 router_tokens_per_expert=[4026, 3053, 143, 4095, 0, 0, 0, 2, 4084, 0, 7, 48, 0, 0, 4023, 0, 0, 0, 0, 0, 3, 0, 0, 2104, 0, 0, 0, 1039, 0, 1, 0, 433, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1920, 3680, 1, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[6426, 8060, 4214, 8192, 6114, 8170, 8192, 8192] dup_rows_per_dst=[170, 96, 0, 2922, 98, 3032, 8, 1650] combine_rows_per_src=[6426, 8060, 4214, 8192, 6114, 8170, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62020 payload_rows_remote=53834 dup_rows=3516 dup_rows_remote=3464 dedup_ratio=0.053650 router_tokens_per_expert=[3743, 0, 0, 0, 2956, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 292, 4095, 0, 0, 0, 148, 0, 0, 0, 4096, 2, 5, 0, 0, 0, 0, 0, 0, 0, 3626, 0, 4087, 0, 0, 515, 0, 0, 0, 0, 4093, 26, 0, 0, 0, 0, 0, 0, 0, 0, 980, 0, 0, 4096] payload_rows_per_dst=[7486, 8174, 8190, 8192, 5922, 7678, 8186, 8192] dup_rows_per_dst=[0, 16, 584, 300, 0, 604, 52, 1960] combine_rows_per_src=[7486, 8174, 8190, 8192, 5922, 7678, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=209756160 dispatch_meta_bytes=230176 combine_payload_bytes=209756160 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=209986336 ingress_bytes=260087808 link_bytes=260087808 token_bytes=419742496 weight_movement_bytes=50331648 total_nvlink_bytes=470074144 weight_to_token_ratio=0.119911 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978967704e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55386 payload_rows_remote=47194 dup_rows=10150 dup_rows_remote=8366 dedup_ratio=0.154877 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 4086, 0, 2292, 221, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 10, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1582, 0, 0, 4096] payload_rows_per_dst=[8192, 8192, 6930, 8192, 7502, 8192, 8186, 0] dup_rows_per_dst=[1784, 5026, 0, 170, 6, 0, 3164, 0] combine_rows_per_src=[8192, 8192, 6930, 8192, 7502, 8192, 8186, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=204283904 dispatch_meta_bytes=229376 combine_payload_bytes=204283904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=204513280 ingress_bytes=229449728 link_bytes=229449728 token_bytes=408797184 weight_movement_bytes=25165824 total_nvlink_bytes=433963008 weight_to_token_ratio=0.061561 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.745847316e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=224624640 dispatch_meta_bytes=223648 combine_payload_bytes=224624640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=224848288 ingress_bytes=249790464 link_bytes=249790464 token_bytes=449472928 weight_movement_bytes=25165824 total_nvlink_bytes=474638752 weight_to_token_ratio=0.055990 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=20480 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.900616814e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=201064448 dispatch_meta_bytes=227744 combine_payload_bytes=201064448 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=201292192 ingress_bytes=276561920 link_bytes=276561920 token_bytes=402356640 weight_movement_bytes=75497472 total_nvlink_bytes=477854112 weight_to_token_ratio=0.187638 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.104316662e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202211328 dispatch_meta_bytes=222776 combine_payload_bytes=202211328 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=202434104 ingress_bytes=252542976 link_bytes=252542976 token_bytes=404645432 weight_movement_bytes=50331648 total_nvlink_bytes=454977080 weight_to_token_ratio=0.124385 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.921560250e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=220504064 dispatch_meta_bytes=229192 combine_payload_bytes=220504064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=220733256 ingress_bytes=245669888 link_bytes=245669888 token_bytes=441237320 weight_movement_bytes=25165824 total_nvlink_bytes=466403144 weight_to_token_ratio=0.057035 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.869263992e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=193306624 dispatch_meta_bytes=222240 combine_payload_bytes=193306624 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193528864 ingress_bytes=218472448 link_bytes=218472448 token_bytes=386835488 weight_movement_bytes=25165824 total_nvlink_bytes=412001312 weight_to_token_ratio=0.065056 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.662322898e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57312 payload_rows_remote=49120 dup_rows=8224 dup_rows_remote=8196 dedup_ratio=0.125488 router_tokens_per_expert=[0, 0, 0, 0, 0, 12, 4096, 0, 0, 593, 0, 0, 0, 0, 52, 0, 0, 197, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2844, 2, 648, 0, 2, 0, 5, 0, 0, 3889, 0, 4087, 0, 0, 4096] payload_rows_per_dst=[8192, 1290, 8192, 8072, 8192, 8192, 7404, 7778] dup_rows_per_dst=[24, 0, 394, 0, 6, 28, 2510, 5262] combine_rows_per_src=[8192, 1290, 8192, 8072, 8192, 8192, 7404, 7778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61888 payload_rows_remote=53696 dup_rows=3648 dup_rows_remote=260 dedup_ratio=0.055664 router_tokens_per_expert=[0, 0, 6, 5, 0, 1, 3718, 0, 3479, 64, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 95, 0, 4083, 0, 0, 0, 0, 4, 4092, 0, 0, 1, 0, 0, 1713, 0, 4073, 4, 0, 0, 0, 0, 5, 37, 0, 0, 0, 0, 0, 3614, 1, 2, 0, 2, 0, 0, 0, 0, 3678, 0, 0, 0, 0, 4086] payload_rows_per_dst=[7454, 7086, 8182, 8190, 8192, 8192, 7236, 7356] dup_rows_per_dst=[6, 10, 174, 4, 3388, 64, 2, 0] combine_rows_per_src=[7454, 7086, 8182, 8190, 8192, 8192, 7236, 7356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53492 payload_rows_remote=48880 dup_rows=12044 dup_rows_remote=12044 dedup_ratio=0.183777 router_tokens_per_expert=[0, 0, 0, 0, 27, 14, 0, 0, 0, 0, 2, 0, 0, 0, 26, 0, 0, 0, 0, 4005, 3702, 0, 0, 0, 0, 0, 0, 0, 2306, 0, 0, 4062, 0, 4095, 0, 0, 4096, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 551, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 334, 0, 3141] payload_rows_per_dst=[8172, 8186, 8192, 4612, 8190, 1102, 8192, 6846] dup_rows_per_dst=[34, 54, 7222, 0, 4604, 0, 0, 130] combine_rows_per_src=[8172, 8186, 8192, 4612, 8190, 1102, 8192, 6846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58186 payload_rows_remote=49994 dup_rows=7350 dup_rows_remote=7028 dedup_ratio=0.112152 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 1025, 2866, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 51, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2814, 0, 0, 4037] payload_rows_per_dst=[8192, 7748, 8192, 8192, 8192, 8074, 7484, 2112] dup_rows_per_dst=[0, 6104, 322, 0, 102, 0, 822, 0] combine_rows_per_src=[8192, 7748, 8192, 8192, 8192, 8074, 7484, 2112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=201195520 dispatch_meta_bytes=229264 combine_payload_bytes=201195520 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201424784 ingress_bytes=226361344 link_bytes=226361344 token_bytes=402620304 weight_movement_bytes=25165824 total_nvlink_bytes=427786128 weight_to_token_ratio=0.062505 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.722348282e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57082 payload_rows_remote=48892 dup_rows=8454 dup_rows_remote=2980 dedup_ratio=0.128998 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4085, 0, 2677, 0, 0, 0, 0, 0, 0, 74, 0, 130, 0, 0, 4, 190, 3966, 0, 4, 8, 0, 0, 4039, 5, 0, 9, 0, 0, 48, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 1, 0, 0, 0, 0, 10, 32, 0, 4081, 0, 1419, 0, 19, 4009] payload_rows_per_dst=[8170, 8190, 8192, 8120, 8190, 8018, 8134, 68] dup_rows_per_dst=[0, 5474, 388, 10, 96, 0, 2452, 34] combine_rows_per_src=[8170, 8190, 8192, 8120, 8190, 8018, 8134, 68] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51726 payload_rows_remote=43542 dup_rows=13810 dup_rows_remote=1272 dedup_ratio=0.210724 router_tokens_per_expert=[20, 0, 0, 0, 0, 0, 3578, 0, 13, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 5, 0, 4093, 0, 3, 0, 0, 0, 635, 0, 4085, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3548, 0, 0, 4093] payload_rows_per_dst=[7196, 8192, 8188, 8192, 3050, 7912, 812, 8184] dup_rows_per_dst=[0, 32, 0, 10, 1230, 0, 0, 12538] combine_rows_per_src=[7196, 8192, 8188, 8192, 3050, 7912, 812, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60140 payload_rows_remote=51952 dup_rows=5396 dup_rows_remote=5396 dedup_ratio=0.082336 router_tokens_per_expert=[4090, 4054, 98, 4095, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 0, 35, 0, 1118, 0, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 3912, 7640, 8170, 8108, 7752, 8188, 8190] dup_rows_per_dst=[196, 30, 0, 5162, 0, 8, 0, 0] combine_rows_per_src=[8180, 3912, 7640, 8170, 8108, 7752, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=219938816 dispatch_meta_bytes=215824 combine_payload_bytes=219938816 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=220154640 ingress_bytes=295436288 link_bytes=295436288 token_bytes=440093456 weight_movement_bytes=75497472 total_nvlink_bytes=515590928 weight_to_token_ratio=0.171549 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.247928794e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56096 payload_rows_remote=47904 dup_rows=9440 dup_rows_remote=8052 dedup_ratio=0.144043 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 2, 0, 0, 4, 0, 0, 1, 4085, 1, 5, 4093, 96, 4082, 0, 8, 4071, 10, 0, 0, 0, 0, 0, 0, 0, 19, 4092, 4089, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 1, 4090, 0, 0] payload_rows_per_dst=[8192, 8178, 8164, 6832, 212, 8190, 8144, 8184] dup_rows_per_dst=[1388, 0, 4984, 3016, 16, 32, 0, 4] combine_rows_per_src=[8192, 8178, 8164, 6832, 212, 8190, 8144, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=200212480 dispatch_meta_bytes=243696 combine_payload_bytes=200212480 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=200456176 ingress_bytes=275709952 link_bytes=275709952 token_bytes=400668656 weight_movement_bytes=75497472 total_nvlink_bytes=476166128 weight_to_token_ratio=0.188429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.097834169e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=204775424 dispatch_meta_bytes=228088 combine_payload_bytes=204775424 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=205003512 ingress_bytes=255107072 link_bytes=255107072 token_bytes=409778936 weight_movement_bytes=50331648 total_nvlink_bytes=460110584 weight_to_token_ratio=0.122826 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.941070058e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=200261632 dispatch_meta_bytes=207488 combine_payload_bytes=200261632 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200469120 ingress_bytes=225427456 link_bytes=225427456 token_bytes=400730752 weight_movement_bytes=25165824 total_nvlink_bytes=425896576 weight_to_token_ratio=0.062800 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=13312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.715242473e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=178348032 dispatch_meta_bytes=179256 combine_payload_bytes=178348032 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=178527288 ingress_bytes=279011328 link_bytes=279011328 token_bytes=356875320 weight_movement_bytes=100663296 total_nvlink_bytes=457538616 weight_to_token_ratio=0.282069 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=20352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.122953826e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=212795392 dispatch_meta_bytes=229392 combine_payload_bytes=212795392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213024784 ingress_bytes=237961216 link_bytes=237961216 token_bytes=425820176 weight_movement_bytes=25165824 total_nvlink_bytes=450986000 weight_to_token_ratio=0.059100 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.810609905e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=196214784 dispatch_meta_bytes=223824 combine_payload_bytes=196214784 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=196438608 ingress_bytes=246546432 link_bytes=246546432 token_bytes=392653392 weight_movement_bytes=50331648 total_nvlink_bytes=442985040 weight_to_token_ratio=0.128183 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.875933479e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64276 payload_rows_remote=56664 dup_rows=1260 dup_rows_remote=964 dedup_ratio=0.019226 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 0, 0, 0, 14, 0, 1, 0, 94, 4020, 0, 2, 0, 0, 0, 0, 0, 3991, 0, 13, 4095, 0, 4081, 2, 15, 4078, 26, 0, 0, 0, 56, 0, 1, 0, 0, 3749, 4094, 0, 2, 0, 0, 0, 0, 1, 11, 0, 5, 0, 2, 0, 4047, 0, 352] payload_rows_per_dst=[8190, 8190, 8084, 8008, 7910, 7612, 8164, 8118] dup_rows_per_dst=[2, 56, 150, 0, 42, 296, 20, 694] combine_rows_per_src=[8190, 8190, 8084, 8008, 7910, 7612, 8164, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63650 payload_rows_remote=55508 dup_rows=1886 dup_rows_remote=1032 dedup_ratio=0.028778 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 56, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 1032, 0, 0, 0, 58, 0, 0, 0, 4096, 0, 62, 3863, 156, 4071, 0, 0, 3570, 271, 0, 0, 0, 0, 3, 0, 0, 0, 4094, 4035, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 4, 0, 0, 4036, 0, 373] payload_rows_per_dst=[7728, 8170, 7910, 8192, 8142, 8188, 7154, 8166] dup_rows_per_dst=[0, 22, 86, 240, 854, 6, 0, 678] combine_rows_per_src=[7728, 8170, 7910, 8192, 8142, 8188, 7154, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62100 payload_rows_remote=57198 dup_rows=3436 dup_rows_remote=3436 dedup_ratio=0.052429 router_tokens_per_expert=[85, 0, 3981, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1559, 0, 0, 0, 0, 0, 892, 0, 4059, 0, 3, 0, 0, 26, 0, 0, 0, 0, 0, 1504, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 176, 2] payload_rows_per_dst=[8120, 8192, 8192, 4902, 8118, 8192, 8192, 8192] dup_rows_per_dst=[12, 0, 0, 0, 58, 3010, 0, 356] combine_rows_per_src=[8120, 8192, 8192, 4902, 8118, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232095744 dispatch_meta_bytes=230512 combine_payload_bytes=232095744 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=232326256 ingress_bytes=282427392 link_bytes=282427392 token_bytes=464422000 weight_movement_bytes=50331648 total_nvlink_bytes=514753648 weight_to_token_ratio=0.108375 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.148946126e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64842 payload_rows_remote=56862 dup_rows=694 dup_rows_remote=694 dedup_ratio=0.010590 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 5, 3985, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 2598, 0, 0, 4096, 1831, 0, 0, 0, 0, 0, 0, 0, 1, 3887, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4077, 0, 7] payload_rows_per_dst=[8166, 8192, 7980, 8192, 8192, 7774, 8192, 8154] dup_rows_per_dst=[0, 10, 0, 0, 666, 2, 0, 16] combine_rows_per_src=[8166, 8192, 7980, 8192, 8192, 7774, 8192, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57328 payload_rows_remote=49158 dup_rows=8208 dup_rows_remote=7552 dedup_ratio=0.125244 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 2, 0, 7, 0, 560, 0, 0, 0, 309, 743, 15, 34, 0, 103, 0, 4, 0, 4065, 0, 4, 3613, 0, 4079, 102, 0, 4047, 3362, 0, 0, 0, 5, 0, 0, 0, 1, 4015, 3844, 0, 0, 20, 25, 0, 1, 0, 0, 51, 0, 1, 59, 36, 3657, 0, 0] payload_rows_per_dst=[7234, 8170, 1930, 8188, 8146, 8042, 8146, 7472] dup_rows_per_dst=[0, 656, 272, 2062, 5042, 0, 40, 136] combine_rows_per_src=[7234, 8170, 1930, 8188, 8146, 8042, 8146, 7472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59156 payload_rows_remote=51280 dup_rows=6380 dup_rows_remote=5232 dedup_ratio=0.097351 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 21, 23, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2470, 2, 0, 0, 45, 0, 1, 0, 3964, 0, 0, 4073, 147, 3171, 0, 41, 4096, 3064, 1, 0, 2, 0, 1, 0, 0, 0, 4095, 4045, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3496, 0, 0] payload_rows_per_dst=[8190, 7458, 4944, 7966, 6338, 8192, 8192, 7876] dup_rows_per_dst=[48, 2, 0, 54, 168, 3862, 1098, 1148] combine_rows_per_src=[8190, 7458, 4944, 7966, 6338, 8192, 8192, 7876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57504 payload_rows_remote=55348 dup_rows=8032 dup_rows_remote=8032 dedup_ratio=0.122559 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4067, 0, 3431, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 507, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 3574, 0, 0, 4096] payload_rows_per_dst=[8134, 6902, 8192, 8192, 8192, 8192, 2156, 7544] dup_rows_per_dst=[0, 44, 2, 0, 1010, 0, 0, 6976] combine_rows_per_src=[8134, 6902, 8192, 8192, 8192, 8192, 2156, 7544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=227360768 dispatch_meta_bytes=226160 combine_payload_bytes=227360768 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=227586928 ingress_bytes=378355712 link_bytes=378355712 token_bytes=454947696 weight_movement_bytes=150994944 total_nvlink_bytes=605942640 weight_to_token_ratio=0.331895 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17920 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.878849802e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59052 payload_rows_remote=50860 dup_rows=6484 dup_rows_remote=6066 dedup_ratio=0.098938 router_tokens_per_expert=[5, 0, 4096, 0, 0, 0, 116, 4069, 0, 0, 0, 4096, 0, 27, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2049, 0, 0, 0, 123, 5, 2913, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 8192, 8184, 8192, 8180, 2130, 8138, 7844] dup_rows_per_dst=[418, 56, 0, 1988, 4022, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8184, 8192, 8180, 2130, 8138, 7844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=234283008 dispatch_meta_bytes=242536 combine_payload_bytes=234283008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234525544 ingress_bytes=259448832 link_bytes=259448832 token_bytes=468808552 weight_movement_bytes=25165824 total_nvlink_bytes=493974376 weight_to_token_ratio=0.053680 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.974105835e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232906752 dispatch_meta_bytes=230224 combine_payload_bytes=232906752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233136976 ingress_bytes=258072576 link_bytes=258072576 token_bytes=466043728 weight_movement_bytes=25165824 total_nvlink_bytes=491209552 weight_to_token_ratio=0.053999 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22528 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.963634117e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=201351168 dispatch_meta_bytes=226840 combine_payload_bytes=201351168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201578008 ingress_bytes=226516992 link_bytes=226516992 token_bytes=402929176 weight_movement_bytes=25165824 total_nvlink_bytes=428095000 weight_to_token_ratio=0.062457 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.723532583e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=210042880 dispatch_meta_bytes=226048 combine_payload_bytes=210042880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210268928 ingress_bytes=235208704 link_bytes=235208704 token_bytes=420311808 weight_movement_bytes=25165824 total_nvlink_bytes=445477632 weight_to_token_ratio=0.059874 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789666469e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=226705408 dispatch_meta_bytes=253520 combine_payload_bytes=226705408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=226958928 ingress_bytes=251871232 link_bytes=251871232 token_bytes=453664336 weight_movement_bytes=25165824 total_nvlink_bytes=478830160 weight_to_token_ratio=0.055472 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.916449054e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=208322560 dispatch_meta_bytes=227704 combine_payload_bytes=208322560 prefetch_bytes=25165824 reduce_bytes=125829120 egress_bytes=208550264 ingress_bytes=359317504 link_bytes=359317504 token_bytes=416872824 weight_movement_bytes=150994944 total_nvlink_bytes=567867768 weight_to_token_ratio=0.362209 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=30208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.733991037e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63060 payload_rows_remote=54868 dup_rows=2476 dup_rows_remote=2438 dedup_ratio=0.037781 router_tokens_per_expert=[62, 0, 4096, 17, 0, 0, 1092, 4071, 0, 0, 0, 4032, 0, 0, 0, 0, 11, 0, 0, 0, 5, 0, 0, 4096, 0, 3175, 0, 0, 1, 0, 55, 45, 0, 3735, 0, 0, 28, 0, 0, 0, 0, 10, 0, 1, 21, 0, 0, 4083, 2, 0, 0, 0, 0, 5, 0, 0, 3, 4094, 0, 0, 0, 5, 23, 0] payload_rows_per_dst=[8192, 8064, 8192, 6546, 7526, 8192, 8156, 8192] dup_rows_per_dst=[2342, 0, 32, 6, 0, 38, 0, 58] combine_rows_per_src=[8192, 8064, 8192, 6546, 7526, 8192, 8156, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61118 payload_rows_remote=52994 dup_rows=4418 dup_rows_remote=474 dedup_ratio=0.067413 router_tokens_per_expert=[0, 40, 4039, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4, 0, 0, 0, 0, 2012, 0, 2206, 0, 1, 0, 0, 3827, 0, 0, 0, 18, 0, 164, 1, 0, 4053, 0, 33, 0, 0, 2, 0, 17, 0, 0, 4077, 0, 0, 0, 5, 0, 7] payload_rows_per_dst=[8096, 8190, 8142, 4032, 8124, 8170, 8192, 8172] dup_rows_per_dst=[62, 0, 0, 0, 3944, 302, 104, 6] combine_rows_per_src=[8096, 8190, 8142, 4032, 8124, 8170, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224739328 dispatch_meta_bytes=229224 combine_payload_bytes=224739328 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=224968552 ingress_bytes=275070976 link_bytes=275070976 token_bytes=449707880 weight_movement_bytes=50331648 total_nvlink_bytes=500039528 weight_to_token_ratio=0.111921 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.092972300e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57122 payload_rows_remote=48944 dup_rows=8414 dup_rows_remote=8414 dedup_ratio=0.128387 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 4094, 9, 5, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 199, 0, 0, 0, 0, 0, 0, 0, 182, 0, 3922, 0, 6, 0, 1, 39, 0, 0, 0, 0, 0, 0, 150, 0, 4096, 0, 0, 4089, 3806, 0, 4091] payload_rows_per_dst=[8192, 7616, 8192, 8178, 8192, 8182, 388, 8182] dup_rows_per_dst=[0, 2, 7974, 0, 398, 40, 0, 0] combine_rows_per_src=[8192, 7616, 8192, 8178, 8192, 8182, 388, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64984 payload_rows_remote=56792 dup_rows=552 dup_rows_remote=552 dedup_ratio=0.008423 router_tokens_per_expert=[3, 0, 4096, 0, 0, 0, 1, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3864, 0, 0, 0, 35, 0, 448, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7668, 8192, 8170, 8186] dup_rows_per_dst=[8, 0, 0, 502, 0, 42, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7668, 8192, 8170, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58472 payload_rows_remote=50316 dup_rows=7064 dup_rows_remote=7064 dedup_ratio=0.107788 router_tokens_per_expert=[0, 41, 3540, 17, 0, 0, 981, 4051, 0, 0, 0, 4075, 3, 0, 0, 0, 0, 0, 18, 0, 0, 0, 2, 4038, 0, 4089, 3, 0, 0, 3, 6, 2981, 39, 653, 0, 0, 61, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4031, 0, 2, 0, 0, 22, 0, 0, 47, 0, 4040, 2, 0, 19, 0, 0, 0] payload_rows_per_dst=[8192, 8156, 8116, 8192, 1442, 8064, 8192, 8118] dup_rows_per_dst=[966, 0, 0, 5972, 70, 0, 52, 4] combine_rows_per_src=[8192, 8156, 8116, 8192, 1442, 8064, 8192, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61336 payload_rows_remote=53194 dup_rows=4200 dup_rows_remote=4126 dedup_ratio=0.064087 router_tokens_per_expert=[29, 0, 3982, 0, 0, 0, 20, 4056, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 77, 0, 8, 0, 0, 0, 4096, 0, 3960, 1, 0, 0, 0, 0, 295, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 62, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 21, 3, 0, 0, 0] payload_rows_per_dst=[4548, 7836, 8124, 8192, 8190, 8192, 8112, 8142] dup_rows_per_dst=[4, 460, 170, 762, 2, 2692, 36, 74] combine_rows_per_src=[4548, 7836, 8124, 8192, 8190, 8192, 8112, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64492 payload_rows_remote=56300 dup_rows=1044 dup_rows_remote=806 dedup_ratio=0.015930 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 3940, 0, 4096, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 7, 0, 14, 2, 0, 8, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 4092, 4096, 0, 4086] payload_rows_per_dst=[8192, 8192, 8164, 8184, 7460, 8174, 8192, 7934] dup_rows_per_dst=[238, 22, 718, 0, 16, 0, 50, 0] combine_rows_per_src=[8192, 8192, 8164, 8184, 7460, 8174, 8192, 7934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217063424 dispatch_meta_bytes=213872 combine_payload_bytes=217063424 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=217277296 ingress_bytes=242229248 link_bytes=242229248 token_bytes=434340720 weight_movement_bytes=25165824 total_nvlink_bytes=459506544 weight_to_token_ratio=0.057940 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.843084697e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=200474624 dispatch_meta_bytes=229432 combine_payload_bytes=200474624 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200704056 ingress_bytes=225640448 link_bytes=225640448 token_bytes=401178680 weight_movement_bytes=25165824 total_nvlink_bytes=426344504 weight_to_token_ratio=0.062730 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.716863096e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61428 payload_rows_remote=53236 dup_rows=4108 dup_rows_remote=4106 dedup_ratio=0.062683 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 2518, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 2041, 0, 0, 4096, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 3956, 0, 0] payload_rows_per_dst=[8192, 8192, 5560, 8174, 7014, 8190, 8192, 7914] dup_rows_per_dst=[0, 0, 58, 0, 0, 4048, 2, 0] combine_rows_per_src=[8192, 8192, 5560, 8174, 7014, 8190, 8192, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=232620032 dispatch_meta_bytes=229376 combine_payload_bytes=232620032 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=232849408 ingress_bytes=257785856 link_bytes=257785856 token_bytes=465469440 weight_movement_bytes=25165824 total_nvlink_bytes=490635264 weight_to_token_ratio=0.054065 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.961452509e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=206094336 dispatch_meta_bytes=229520 combine_payload_bytes=206094336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206323856 ingress_bytes=231260160 link_bytes=231260160 token_bytes=412418192 weight_movement_bytes=25165824 total_nvlink_bytes=437584016 weight_to_token_ratio=0.061020 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.759622611e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217882624 dispatch_meta_bytes=229280 combine_payload_bytes=217882624 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=218111904 ingress_bytes=268214272 link_bytes=268214272 token_bytes=435994528 weight_movement_bytes=50331648 total_nvlink_bytes=486326176 weight_to_token_ratio=0.115441 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.040800705e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=230604800 dispatch_meta_bytes=228424 combine_payload_bytes=230604800 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=230833224 ingress_bytes=280936448 link_bytes=280936448 token_bytes=461438024 weight_movement_bytes=50331648 total_nvlink_bytes=511769672 weight_to_token_ratio=0.109076 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.137601765e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60112 payload_rows_remote=56152 dup_rows=5424 dup_rows_remote=4802 dedup_ratio=0.082764 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 9, 0, 14, 0, 102, 0, 0, 0, 0, 0, 2739, 0, 4057, 0, 0, 0, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 53, 51, 0, 0, 1, 0, 0, 177, 0, 0, 1587, 0, 524, 3, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4092, 3988, 59, 3599] payload_rows_per_dst=[8192, 8182, 7968, 8184, 8192, 3960, 8118, 7316] dup_rows_per_dst=[22, 26, 4556, 0, 198, 622, 0, 0] combine_rows_per_src=[8192, 8182, 7968, 8184, 8192, 3960, 8118, 7316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=218054656 dispatch_meta_bytes=229368 combine_payload_bytes=218054656 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=218284024 ingress_bytes=268386304 link_bytes=268386304 token_bytes=436338680 weight_movement_bytes=50331648 total_nvlink_bytes=486670328 weight_to_token_ratio=0.115350 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.042109670e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64290 payload_rows_remote=56098 dup_rows=1246 dup_rows_remote=1214 dedup_ratio=0.019012 router_tokens_per_expert=[0, 8, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 2, 32, 0, 4055, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4087, 0, 0, 0, 209, 90, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 4084, 4078, 5, 3773] payload_rows_per_dst=[8012, 8160, 7880, 8168, 8192, 8192, 8130, 7556] dup_rows_per_dst=[16, 562, 56, 0, 32, 420, 160, 0] combine_rows_per_src=[8012, 8160, 7880, 8168, 8192, 8192, 8130, 7556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59372 payload_rows_remote=51968 dup_rows=6164 dup_rows_remote=6162 dedup_ratio=0.094055 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 150, 0, 4, 0, 4095, 0, 0, 3689, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 0, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8182, 8192, 8192, 7404, 8192, 8188, 2832, 8190] dup_rows_per_dst=[0, 6, 306, 2, 5850, 0, 0, 0] combine_rows_per_src=[8182, 8192, 8192, 7404, 8192, 8188, 2832, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=229998592 dispatch_meta_bytes=243816 combine_payload_bytes=229998592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230242408 ingress_bytes=255164416 link_bytes=255164416 token_bytes=460241000 weight_movement_bytes=25165824 total_nvlink_bytes=485406824 weight_to_token_ratio=0.054680 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=29824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.941506380e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65424 payload_rows_remote=57330 dup_rows=112 dup_rows_remote=104 dedup_ratio=0.001709 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 4091, 0, 0, 1, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 45, 4094] payload_rows_per_dst=[8192, 8192, 8094, 8192, 8192, 8192, 8182, 8188] dup_rows_per_dst=[0, 0, 8, 0, 0, 14, 0, 90] combine_rows_per_src=[8192, 8192, 8094, 8192, 8192, 8192, 8182, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62680 payload_rows_remote=54494 dup_rows=2856 dup_rows_remote=2694 dedup_ratio=0.043579 router_tokens_per_expert=[0, 0, 8, 2, 0, 0, 0, 0, 13, 0, 0, 0, 155, 0, 0, 0, 1727, 0, 3970, 0, 0, 0, 1450, 1, 36, 0, 3, 0, 0, 0, 28, 1, 4058, 0, 0, 0, 0, 0, 0, 55, 733, 0, 0, 3857, 0, 116, 0, 650, 0, 0, 0, 0, 0, 0, 86, 0, 0, 4035, 0, 0, 4095, 4006, 43, 3640] payload_rows_per_dst=[8086, 8186, 6356, 8192, 8192, 8190, 8112, 7366] dup_rows_per_dst=[4, 162, 0, 134, 34, 2522, 0, 0] combine_rows_per_src=[8086, 8186, 6356, 8192, 8192, 8190, 8112, 7366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57026 payload_rows_remote=48868 dup_rows=8510 dup_rows_remote=5924 dedup_ratio=0.129852 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7, 1, 0, 0, 1, 4051, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 52, 7, 0, 0, 0, 0, 45, 0, 1, 4085, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 49, 4088, 0, 0, 4094, 4092, 1, 4031] payload_rows_per_dst=[8192, 8076, 0, 8112, 8134, 8172, 8182, 8158] dup_rows_per_dst=[6, 10, 0, 0, 112, 148, 5648, 2586] combine_rows_per_src=[8192, 8076, 0, 8112, 8134, 8172, 8182, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=229777408 dispatch_meta_bytes=229248 combine_payload_bytes=229777408 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=230006656 ingress_bytes=280109056 link_bytes=280109056 token_bytes=459784064 weight_movement_bytes=50331648 total_nvlink_bytes=510115712 weight_to_token_ratio=0.109468 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.131306268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54604 payload_rows_remote=46412 dup_rows=10932 dup_rows_remote=10422 dedup_ratio=0.166809 router_tokens_per_expert=[0, 0, 0, 0, 31, 0, 0, 0, 4091, 129, 0, 0, 0, 1230, 0, 0, 0, 0, 0, 947, 0, 1, 0, 3962, 1, 0, 3422, 1, 73, 0, 0, 0, 0, 0, 0, 0, 1, 9, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 0, 0, 0, 0, 0, 4094, 0, 4089, 7, 0, 0, 3875, 1, 0] payload_rows_per_dst=[8192, 7708, 8178, 6846, 8192, 8182, 4, 7302] dup_rows_per_dst=[510, 1586, 3248, 148, 5438, 0, 0, 2] combine_rows_per_src=[8192, 7708, 8178, 6846, 8192, 8182, 4, 7302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=212860928 dispatch_meta_bytes=232520 combine_payload_bytes=212860928 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213093448 ingress_bytes=238026752 link_bytes=238026752 token_bytes=425954376 weight_movement_bytes=25165824 total_nvlink_bytes=451120200 weight_to_token_ratio=0.059081 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18304 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.811108558e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65402 payload_rows_remote=57210 dup_rows=134 dup_rows_remote=134 dedup_ratio=0.002045 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 44, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8058, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 128, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8058, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234823680 dispatch_meta_bytes=229736 combine_payload_bytes=234823680 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=235053416 ingress_bytes=335486976 link_bytes=335486976 token_bytes=469877096 weight_movement_bytes=100663296 total_nvlink_bytes=570540392 weight_to_token_ratio=0.214233 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=18944 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.552668254e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=223207424 dispatch_meta_bytes=228752 combine_payload_bytes=223207424 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=223436176 ingress_bytes=248373248 link_bytes=248373248 token_bytes=446643600 weight_movement_bytes=25165824 total_nvlink_bytes=471809424 weight_to_token_ratio=0.056344 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.889833438e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=200163328 dispatch_meta_bytes=219168 combine_payload_bytes=200163328 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=200382496 ingress_bytes=325992448 link_bytes=325992448 token_bytes=400545824 weight_movement_bytes=125829120 total_nvlink_bytes=526374944 weight_to_token_ratio=0.314144 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.480425866e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49722 payload_rows_remote=41566 dup_rows=15814 dup_rows_remote=15132 dedup_ratio=0.241302 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 16, 3959, 148, 0, 0, 0, 4074, 0, 0, 9, 0, 0, 0, 0, 11, 0, 3611, 0, 0, 3783, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1043, 0, 52, 1, 0, 0, 395, 0, 12, 0, 0, 4086, 0, 0, 0, 0, 0, 4071, 0, 4096, 0, 0, 12, 3356, 0, 0] payload_rows_per_dst=[8192, 8170, 40, 8154, 2102, 8156, 8172, 6736] dup_rows_per_dst=[36, 274, 0, 6648, 32, 682, 8142, 0] combine_rows_per_src=[8192, 8170, 40, 8154, 2102, 8156, 8172, 6736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=190103552 dispatch_meta_bytes=227336 combine_payload_bytes=190103552 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=190330888 ingress_bytes=240435200 link_bytes=240435200 token_bytes=380434440 weight_movement_bytes=50331648 total_nvlink_bytes=430766088 weight_to_token_ratio=0.132300 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.829434064e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=234332160 dispatch_meta_bytes=229376 combine_payload_bytes=234332160 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=234561536 ingress_bytes=284663808 link_bytes=284663808 token_bytes=468893696 weight_movement_bytes=50331648 total_nvlink_bytes=519225344 weight_to_token_ratio=0.107341 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.165962668e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52126 payload_rows_remote=50742 dup_rows=13410 dup_rows_remote=13210 dedup_ratio=0.204620 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8, 4057, 660, 0, 0, 0, 4039, 0, 0, 0, 0, 5, 2805, 1, 0, 12, 4063, 0, 0, 1589, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 0, 0, 0, 0, 0, 4096, 0, 4089, 14, 0, 0, 3791, 0, 3] payload_rows_per_dst=[8192, 8192, 6074, 5486, 1384, 8114, 7074, 7610] dup_rows_per_dst=[2, 1206, 3240, 2156, 200, 0, 6600, 6] combine_rows_per_src=[8192, 8192, 6074, 5486, 1384, 8114, 7074, 7610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61708 payload_rows_remote=53516 dup_rows=3828 dup_rows_remote=3826 dedup_ratio=0.058411 router_tokens_per_expert=[3154, 0, 117, 6, 0, 1411, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4085, 6, 0, 0, 0, 0, 120, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3620, 499, 3197, 0, 0, 0, 0, 4, 0, 0, 0, 36] payload_rows_per_dst=[7260, 8046, 8192, 8192, 8192, 6394, 7240, 8192] dup_rows_per_dst=[2116, 786, 0, 2, 664, 0, 182, 78] combine_rows_per_src=[7260, 8046, 8192, 8192, 8192, 6394, 7240, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=170254336 dispatch_meta_bytes=226792 combine_payload_bytes=170254336 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=170481128 ingress_bytes=220585984 link_bytes=220585984 token_bytes=340735464 weight_movement_bytes=50331648 total_nvlink_bytes=391067112 weight_to_token_ratio=0.147715 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.678404465e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55616 payload_rows_remote=47424 dup_rows=9920 dup_rows_remote=6820 dedup_ratio=0.151367 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 65, 0, 0, 0, 3722, 0, 0, 0, 0, 0, 1550, 0, 2, 0, 4094, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 4025, 0, 0] payload_rows_per_dst=[8184, 7574, 8192, 476, 8192, 8192, 6756, 8050] dup_rows_per_dst=[0, 0, 3100, 0, 6820, 0, 0, 0] combine_rows_per_src=[8184, 7574, 8192, 476, 8192, 8192, 6756, 8050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51950 payload_rows_remote=50674 dup_rows=13586 dup_rows_remote=13586 dedup_ratio=0.207306 router_tokens_per_expert=[0, 13, 7, 3, 8, 0, 3, 0, 2570, 606, 0, 0, 0, 32, 0, 0, 0, 0, 26, 31, 0, 0, 1, 4078, 0, 1, 4095, 4, 67, 0, 0, 314, 0, 120, 0, 0, 0, 0, 4055, 30, 0, 0, 0, 0, 0, 8, 0, 4, 68, 4022, 0, 523, 0, 0, 0, 4084, 0, 3906, 18, 0, 0, 4071, 0, 0] payload_rows_per_dst=[7844, 1276, 8188, 8192, 8190, 5164, 4918, 8178] dup_rows_per_dst=[36, 0, 4278, 770, 8388, 0, 114, 0] combine_rows_per_src=[7844, 1276, 8188, 8192, 8190, 5164, 4918, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=207839232 dispatch_meta_bytes=255808 combine_payload_bytes=207839232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=208095040 ingress_bytes=233005056 link_bytes=233005056 token_bytes=415934272 weight_movement_bytes=25165824 total_nvlink_bytes=441100096 weight_to_token_ratio=0.060504 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.772899254e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=219201536 dispatch_meta_bytes=229368 combine_payload_bytes=219201536 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=219430904 ingress_bytes=244367360 link_bytes=244367360 token_bytes=438632440 weight_movement_bytes=25165824 total_nvlink_bytes=463798264 weight_to_token_ratio=0.057373 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.859353259e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57058 payload_rows_remote=48874 dup_rows=8478 dup_rows_remote=8386 dedup_ratio=0.129364 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 4086, 3526, 0, 0, 0, 4069, 32, 0, 0, 0, 0, 4091, 0, 2, 0, 4095, 0, 0, 440, 0, 46, 0, 0, 14, 0, 0, 0, 2, 0, 0, 118, 0, 0, 6, 3, 0, 0, 1, 0, 10, 0, 46, 0, 0, 3, 0, 0, 4086, 0, 3998, 0, 0, 0, 4092, 0, 0] payload_rows_per_dst=[8000, 7892, 8182, 8192, 240, 8190, 8178, 8184] dup_rows_per_dst=[0, 5358, 4, 998, 0, 2026, 0, 92] combine_rows_per_src=[8000, 7892, 8182, 8192, 240, 8190, 8178, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62780 payload_rows_remote=54594 dup_rows=2756 dup_rows_remote=568 dedup_ratio=0.042053 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3377, 0, 4092, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4074, 0, 4080] payload_rows_per_dst=[8192, 8148, 5518, 8192, 8192, 8192, 8186, 8160] dup_rows_per_dst=[0, 0, 52, 0, 0, 516, 2188, 0] combine_rows_per_src=[8192, 8148, 5518, 8192, 8192, 8192, 8186, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=194248704 dispatch_meta_bytes=216976 combine_payload_bytes=194248704 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=194465680 ingress_bytes=244580352 link_bytes=244580352 token_bytes=388714384 weight_movement_bytes=50331648 total_nvlink_bytes=439046032 weight_to_token_ratio=0.129482 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.860973882e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53600 payload_rows_remote=50992 dup_rows=11936 dup_rows_remote=11468 dedup_ratio=0.182129 router_tokens_per_expert=[159, 0, 1073, 0, 1, 66, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 4014, 4096, 0, 0, 0, 0, 3, 0, 0, 1, 0, 4093, 0, 1, 5, 4086, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 4045, 4092, 0, 0, 2, 0, 40, 0, 0, 0, 0] payload_rows_per_dst=[2608, 8120, 8192, 7720, 8176, 8192, 4716, 5876] dup_rows_per_dst=[468, 7462, 2238, 2, 4, 1758, 0, 4] combine_rows_per_src=[2608, 8120, 8192, 7720, 8176, 8192, 4716, 5876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=207560704 dispatch_meta_bytes=257040 combine_payload_bytes=207560704 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=207817744 ingress_bytes=283058176 link_bytes=283058176 token_bytes=415378448 weight_movement_bytes=75497472 total_nvlink_bytes=490875920 weight_to_token_ratio=0.181756 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24320 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.153745664e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6450521240 prefetch_bytes=402653184 reduce_bytes=176160768 weight_movement_bytes=578813952 egress_bytes=3227083928 ingress_bytes=3802251264 link_bytes=3802251264 total_nvlink_bytes=7029335192 weight_to_token_ratio=0.089731 prefetch_share_of_ingress=0.105899 dedup_ratio=0.138279 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=7 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040211 gemm_inner_pad_upper_bound=17399 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.808171179e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56626 payload_rows_remote=48528 dup_rows=8910 dup_rows_remote=8910 dedup_ratio=0.135956 router_tokens_per_expert=[265, 0, 2608, 0, 0, 627, 0, 0, 0, 0, 1263, 4, 0, 0, 0, 0, 0, 4093, 4096, 61, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 3208, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 4035, 0, 0, 0, 0, 59, 0, 0, 0, 137] payload_rows_per_dst=[6802, 2534, 8192, 8192, 6426, 8098, 8192, 8190] dup_rows_per_dst=[198, 0, 126, 24, 0, 0, 8174, 388] combine_rows_per_src=[6802, 2534, 8192, 8192, 6426, 8098, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=200187904 dispatch_meta_bytes=229040 combine_payload_bytes=200187904 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=200416944 ingress_bytes=275685376 link_bytes=275685376 token_bytes=400604848 weight_movement_bytes=75497472 total_nvlink_bytes=476102320 weight_to_token_ratio=0.188459 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.097647175e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=223617024 dispatch_meta_bytes=220648 combine_payload_bytes=223617024 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=223837672 ingress_bytes=273948672 link_bytes=273948672 token_bytes=447454696 weight_movement_bytes=50331648 total_nvlink_bytes=497786344 weight_to_token_ratio=0.112484 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.084432864e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=208863232 dispatch_meta_bytes=249840 combine_payload_bytes=208863232 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=209113072 ingress_bytes=284360704 link_bytes=284360704 token_bytes=417976304 weight_movement_bytes=75497472 total_nvlink_bytes=493473776 weight_to_token_ratio=0.180626 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.163656397e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60990 payload_rows_remote=53232 dup_rows=4546 dup_rows_remote=4546 dedup_ratio=0.069366 router_tokens_per_expert=[4087, 41, 3, 0, 0, 1250, 0, 11, 10, 94, 3148, 0, 0, 0, 126, 0, 0, 3924, 4048, 951, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 3866, 0, 0, 0, 0, 7, 0, 6, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 9, 17, 8, 2851, 4095, 0, 0, 116, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6690, 8190, 8142, 7758, 8192, 5746, 8080] dup_rows_per_dst=[2592, 66, 1808, 0, 0, 56, 24, 0] combine_rows_per_src=[8192, 6690, 8190, 8142, 7758, 8192, 5746, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=198770688 dispatch_meta_bytes=229752 combine_payload_bytes=198770688 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=199000440 ingress_bytes=249102336 link_bytes=249102336 token_bytes=397771128 weight_movement_bytes=50331648 total_nvlink_bytes=448102776 weight_to_token_ratio=0.126534 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.895380955e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6018262776 prefetch_bytes=402653184 reduce_bytes=679477248 weight_movement_bytes=1082130432 egress_bytes=3010881272 ingress_bytes=4089511936 link_bytes=4089511936 total_nvlink_bytes=7100393208 weight_to_token_ratio=0.179808 prefetch_share_of_ingress=0.098460 dedup_ratio=0.184849 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=27 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040211 gemm_inner_pad_upper_bound=17145 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.944778791e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57574 payload_rows_remote=49382 dup_rows=7962 dup_rows_remote=7754 dedup_ratio=0.121490 router_tokens_per_expert=[4013, 0, 3241, 0, 0, 0, 0, 0, 0, 247, 460, 0, 0, 0, 0, 0, 0, 4095, 4093, 107, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8192, 7968, 8192, 460, 8190] dup_rows_per_dst=[6316, 1412, 208, 0, 0, 26, 0, 0] combine_rows_per_src=[8192, 8188, 8192, 8192, 7968, 8192, 460, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57146 payload_rows_remote=50028 dup_rows=8390 dup_rows_remote=3692 dedup_ratio=0.128021 router_tokens_per_expert=[40, 2, 3481, 514, 0, 0, 8, 0, 0, 170, 2173, 4, 29, 0, 0, 36, 0, 4050, 3783, 2, 20, 0, 8, 0, 49, 1, 0, 0, 4082, 162, 0, 0, 4031, 0, 0, 0, 0, 0, 637, 0, 13, 0, 0, 0, 0, 0, 2, 46, 0, 0, 1, 1, 849, 3496, 4062, 0, 1, 1, 0, 944, 0, 0, 0, 70] payload_rows_per_dst=[7978, 7118, 7622, 8192, 8188, 8192, 1702, 8154] dup_rows_per_dst=[112, 4698, 4, 396, 1148, 54, 0, 1978] combine_rows_per_src=[7978, 7118, 7622, 8192, 8188, 8192, 1702, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=218038272 dispatch_meta_bytes=231112 combine_payload_bytes=218038272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=218269384 ingress_bytes=243204096 link_bytes=243204096 token_bytes=436307656 weight_movement_bytes=25165824 total_nvlink_bytes=461473480 weight_to_token_ratio=0.057679 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=26624 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.850502164e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6520862048 prefetch_bytes=452984832 reduce_bytes=478150656 weight_movement_bytes=931135488 egress_bytes=3262256480 ingress_bytes=4189741056 link_bytes=4189741056 total_nvlink_bytes=7451997536 weight_to_token_ratio=0.142793 prefetch_share_of_ingress=0.108118 dedup_ratio=0.134645 prefetch_slots=36 unused_prefetch_slots=220 slot_utilization=0.140625 reduce_slots=19 remote_demand=18 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039982 gemm_inner_pad_upper_bound=17526 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.992443029e-04 prefetch_bytes_per_gemm_flop=2.154181985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58830 payload_rows_remote=50640 dup_rows=6706 dup_rows_remote=6690 dedup_ratio=0.102325 router_tokens_per_expert=[4079, 0, 3258, 0, 0, 2, 0, 4, 0, 167, 3741, 0, 0, 5, 0, 0, 0, 4078, 4094, 25, 0, 0, 0, 0, 54, 0, 0, 0, 4091, 0, 0, 0, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 3660, 4088, 2, 0, 0, 13, 0, 0, 0, 2, 10] payload_rows_per_dst=[6458, 7526, 4342, 8192, 8130, 8188, 7804, 8190] dup_rows_per_dst=[2, 300, 50, 98, 1158, 2, 5080, 16] combine_rows_per_src=[6458, 7526, 4342, 8192, 8130, 8188, 7804, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55002 payload_rows_remote=46810 dup_rows=10534 dup_rows_remote=5980 dedup_ratio=0.160736 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 2155, 0, 2, 0, 4060, 0, 0, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 0, 210, 0, 0, 0, 15, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 3801, 0, 0] payload_rows_per_dst=[8166, 8102, 4312, 8120, 2316, 8192, 8192, 7602] dup_rows_per_dst=[0, 0, 2, 1944, 0, 450, 4554, 3584] combine_rows_per_src=[8166, 8102, 4312, 8120, 2316, 8192, 8192, 7602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=202268672 dispatch_meta_bytes=228544 combine_payload_bytes=202268672 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=202497216 ingress_bytes=277766144 link_bytes=277766144 token_bytes=404765888 weight_movement_bytes=75497472 total_nvlink_bytes=480263360 weight_to_token_ratio=0.186521 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.113479415e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=204914688 dispatch_meta_bytes=214880 combine_payload_bytes=204914688 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=205129568 ingress_bytes=230080512 link_bytes=230080512 token_bytes=410044256 weight_movement_bytes=25165824 total_nvlink_bytes=435210080 weight_to_token_ratio=0.061373 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.750646853e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6687323512 prefetch_bytes=301989888 reduce_bytes=553648128 weight_movement_bytes=855638016 egress_bytes=3345479032 ingress_bytes=4197482496 link_bytes=4197482496 total_nvlink_bytes=7542961528 weight_to_token_ratio=0.127949 prefetch_share_of_ingress=0.071945 dedup_ratio=0.110844 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=22 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=16891 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.996124492e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=207421440 dispatch_meta_bytes=229320 combine_payload_bytes=207421440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=207650760 ingress_bytes=232587264 link_bytes=232587264 token_bytes=415072200 weight_movement_bytes=25165824 total_nvlink_bytes=440238024 weight_to_token_ratio=0.060630 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.769720339e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=191733760 dispatch_meta_bytes=211160 combine_payload_bytes=191733760 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=191944920 ingress_bytes=292397056 link_bytes=292397056 token_bytes=383678680 weight_movement_bytes=100663296 total_nvlink_bytes=484341976 weight_to_token_ratio=0.262364 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.224803750e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6828125880 prefetch_bytes=377487360 reduce_bytes=427819008 weight_movement_bytes=805306368 egress_bytes=3415912120 ingress_bytes=4217520128 link_bytes=4217520128 total_nvlink_bytes=7633432248 weight_to_token_ratio=0.117940 prefetch_share_of_ingress=0.089505 dedup_ratio=0.099911 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=17 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=16764 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.005653444e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6289196992 prefetch_bytes=452984832 reduce_bytes=125829120 weight_movement_bytes=578813952 egress_bytes=3146368960 ingress_bytes=3721641984 link_bytes=3721641984 total_nvlink_bytes=6868010944 weight_to_token_ratio=0.092033 prefetch_share_of_ingress=0.121716 dedup_ratio=0.154388 prefetch_slots=36 unused_prefetch_slots=220 slot_utilization=0.140625 reduce_slots=5 remote_demand=18 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=17653 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.769837211e-04 prefetch_bytes_per_gemm_flop=2.154181985e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5952623112 prefetch_bytes=427819008 reduce_bytes=452984832 weight_movement_bytes=880803840 egress_bytes=2978042376 ingress_bytes=3855384576 link_bytes=3855384576 total_nvlink_bytes=6833426952 weight_to_token_ratio=0.147969 prefetch_share_of_ingress=0.110967 dedup_ratio=0.184830 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=18 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039982 gemm_inner_pad_upper_bound=17653 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.833438873e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53398 payload_rows_remote=45206 dup_rows=12138 dup_rows_remote=4114 dedup_ratio=0.185211 router_tokens_per_expert=[987, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 0, 4096, 4007, 83, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1900, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4013, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0] payload_rows_per_dst=[4868, 2590, 8116, 8192, 5056, 8192, 8192, 8192] dup_rows_per_dst=[1374, 0, 64, 0, 2668, 0, 8024, 8] combine_rows_per_src=[4868, 2590, 8116, 8192, 5056, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=185163776 dispatch_meta_bytes=197280 combine_payload_bytes=185163776 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=185361056 ingress_bytes=285827072 link_bytes=285827072 token_bytes=370524832 weight_movement_bytes=100663296 total_nvlink_bytes=471188128 weight_to_token_ratio=0.271678 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=15360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.174813763e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6406483824 prefetch_bytes=452984832 reduce_bytes=377487360 weight_movement_bytes=830472192 egress_bytes=3205025648 ingress_bytes=4031930368 link_bytes=4031930368 total_nvlink_bytes=7236956016 weight_to_token_ratio=0.129630 prefetch_share_of_ingress=0.112349 dedup_ratio=0.143314 prefetch_slots=36 unused_prefetch_slots=220 slot_utilization=0.140625 reduce_slots=15 remote_demand=18 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=16383 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.917395716e-04 prefetch_bytes_per_gemm_flop=2.154181985e-05 anomalies=none + [2026-09-21 08:31:45] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 220.9 | throughput per GPU (TFLOP/s/GPU): 145.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.073602E+01 | loss scale: 1.0 | grad norm: 20.803 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46566 payload_rows_remote=40038 dup_rows=18970 dup_rows_remote=15302 dedup_ratio=0.289459 router_tokens_per_expert=[981, 599, 253, 4, 2, 1006, 1289, 151, 0, 1370, 119, 0, 23, 75, 91, 243, 21, 341, 352, 1295, 11, 266, 308, 46, 329, 59, 35, 665, 771, 1462, 305, 1070, 188, 254, 725, 453, 149, 1905, 486, 938, 839, 1255, 1585, 12, 2134, 1252, 5, 250, 784, 491, 1000, 45, 797, 829, 493, 38, 1444, 6, 507, 11, 49, 61, 221, 20] payload_rows_per_dst=[5762, 4340, 5768, 6416, 6528, 5862, 5312, 6578] dup_rows_per_dst=[2808, 1160, 2016, 2976, 3668, 2030, 1984, 2328] combine_rows_per_src=[5762, 4340, 5768, 6416, 6528, 5862, 5312, 6578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46892 payload_rows_remote=41496 dup_rows=18644 dup_rows_remote=16504 dedup_ratio=0.284485 router_tokens_per_expert=[1334, 639, 311, 0, 10, 568, 1585, 122, 73, 2009, 21, 5, 11, 181, 188, 321, 75, 821, 486, 1297, 13, 290, 306, 30, 217, 55, 29, 421, 563, 825, 504, 1154, 374, 224, 612, 578, 234, 1233, 143, 740, 416, 1151, 1016, 66, 2303, 1531, 33, 232, 811, 621, 869, 138, 820, 1006, 447, 49, 1413, 23, 223, 39, 7, 94, 832, 26] payload_rows_per_dst=[6072, 6334, 6678, 5396, 5886, 4844, 4796, 6886] dup_rows_per_dst=[3066, 2700, 3020, 2140, 2390, 984, 1310, 3034] combine_rows_per_src=[6072, 6334, 6678, 5396, 5886, 4844, 4796, 6886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46274 payload_rows_remote=39562 dup_rows=19262 dup_rows_remote=15622 dedup_ratio=0.293915 router_tokens_per_expert=[1082, 571, 183, 5, 6, 576, 1388, 215, 17, 1647, 101, 0, 28, 72, 375, 253, 34, 643, 745, 1934, 22, 369, 155, 10, 172, 26, 49, 422, 780, 969, 302, 1119, 233, 241, 508, 309, 117, 1843, 371, 795, 599, 981, 1255, 34, 1574, 1264, 32, 387, 966, 890, 1019, 15, 584, 1207, 527, 82, 1183, 20, 675, 9, 29, 43, 676, 30] payload_rows_per_dst=[5468, 6176, 6712, 5560, 6080, 5148, 4802, 6328] dup_rows_per_dst=[2584, 3262, 3640, 2118, 2754, 1428, 1326, 2150] combine_rows_per_src=[5468, 6176, 6712, 5560, 6080, 5148, 4802, 6328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45192 payload_rows_remote=39226 dup_rows=20344 dup_rows_remote=18820 dedup_ratio=0.310425 router_tokens_per_expert=[635, 595, 173, 16, 8, 786, 1341, 156, 6, 1971, 81, 0, 5, 213, 114, 646, 11, 634, 497, 1286, 41, 729, 314, 7, 398, 16, 50, 246, 833, 740, 490, 814, 219, 207, 830, 397, 176, 1986, 460, 801, 574, 975, 1240, 21, 2156, 1383, 5, 420, 976, 841, 975, 21, 485, 1242, 843, 94, 784, 8, 406, 4, 29, 35, 210, 113] payload_rows_per_dst=[3862, 5918, 5360, 5268, 5174, 6664, 6980, 5966] dup_rows_per_dst=[876, 2638, 1678, 1914, 2230, 4022, 5462, 1524] combine_rows_per_src=[3862, 5918, 5360, 5268, 5174, 6664, 6980, 5966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46450 payload_rows_remote=40258 dup_rows=19086 dup_rows_remote=16618 dedup_ratio=0.291229 router_tokens_per_expert=[1083, 590, 118, 6, 9, 963, 1316, 131, 18, 1626, 14, 1, 58, 235, 102, 258, 2, 692, 520, 1588, 14, 394, 170, 37, 542, 31, 34, 357, 931, 1090, 496, 868, 121, 156, 792, 429, 82, 1648, 377, 968, 457, 1130, 955, 28, 1802, 1599, 28, 247, 1012, 651, 857, 27, 890, 993, 534, 61, 1595, 44, 445, 24, 15, 117, 368, 22] payload_rows_per_dst=[5720, 5034, 5398, 5854, 5882, 6192, 5926, 6444] dup_rows_per_dst=[2712, 1576, 1664, 2844, 2532, 2468, 2870, 2420] combine_rows_per_src=[5720, 5034, 5398, 5854, 5882, 6192, 5926, 6444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45956 payload_rows_remote=39736 dup_rows=19580 dup_rows_remote=16186 dedup_ratio=0.298767 router_tokens_per_expert=[1083, 668, 360, 59, 10, 1028, 1471, 128, 19, 2421, 41, 2, 14, 153, 112, 154, 54, 183, 296, 1356, 17, 307, 243, 85, 309, 127, 28, 575, 636, 1323, 224, 686, 104, 153, 586, 487, 69, 1257, 300, 614, 1167, 841, 1274, 42, 2035, 1627, 24, 171, 764, 581, 971, 122, 735, 1510, 846, 85, 1027, 10, 560, 14, 56, 100, 439, 25] payload_rows_per_dst=[6220, 6870, 6088, 5652, 5390, 6848, 4872, 4016] dup_rows_per_dst=[3394, 3924, 2248, 2164, 1750, 3978, 1394, 728] combine_rows_per_src=[6220, 6870, 6088, 5652, 5390, 6848, 4872, 4016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163995648 dispatch_meta_bytes=221360 combine_payload_bytes=163995648 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=164217008 ingress_bytes=214327296 link_bytes=214327296 token_bytes=328212656 weight_movement_bytes=50331648 total_nvlink_bytes=378544304 weight_to_token_ratio=0.153351 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.630783081e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=169967616 dispatch_meta_bytes=232000 combine_payload_bytes=169967616 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=170199616 ingress_bytes=195133440 link_bytes=195133440 token_bytes=340167232 weight_movement_bytes=25165824 total_nvlink_bytes=365333056 weight_to_token_ratio=0.073981 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.484740014e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162045952 dispatch_meta_bytes=220736 combine_payload_bytes=162045952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=162266688 ingress_bytes=187211776 link_bytes=187211776 token_bytes=324312640 weight_movement_bytes=25165824 total_nvlink_bytes=349478464 weight_to_token_ratio=0.077597 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.424465304e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160669696 dispatch_meta_bytes=232184 combine_payload_bytes=160669696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160901880 ingress_bytes=185835520 link_bytes=185835520 token_bytes=321571576 weight_movement_bytes=25165824 total_nvlink_bytes=346737400 weight_to_token_ratio=0.078259 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.413993586e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164896768 dispatch_meta_bytes=227504 combine_payload_bytes=164896768 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=165124272 ingress_bytes=240394240 link_bytes=240394240 token_bytes=330021040 weight_movement_bytes=75497472 total_nvlink_bytes=405518512 weight_to_token_ratio=0.228766 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.829122406e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162758656 dispatch_meta_bytes=223688 combine_payload_bytes=162758656 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=162982344 ingress_bytes=238256128 link_bytes=238256128 token_bytes=325741000 weight_movement_bytes=75497472 total_nvlink_bytes=401238472 weight_to_token_ratio=0.231771 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=9216 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.812853844e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45824 payload_rows_remote=39142 dup_rows=19712 dup_rows_remote=14888 dedup_ratio=0.300781 router_tokens_per_expert=[1409, 604, 207, 3, 26, 949, 1185, 230, 17, 2023, 14, 2, 18, 74, 225, 107, 49, 512, 420, 1490, 23, 177, 261, 11, 350, 38, 39, 517, 897, 976, 288, 926, 246, 236, 628, 805, 132, 1630, 275, 820, 411, 754, 1353, 14, 1994, 1399, 19, 193, 629, 782, 975, 60, 894, 1099, 757, 26, 1338, 18, 374, 6, 9, 159, 606, 60] payload_rows_per_dst=[5588, 5490, 4746, 6012, 4704, 6042, 6682, 6560] dup_rows_per_dst=[2742, 1668, 1140, 2946, 1580, 2244, 4824, 2568] combine_rows_per_src=[5588, 5490, 4746, 6012, 4704, 6042, 6682, 6560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46334 payload_rows_remote=40674 dup_rows=19202 dup_rows_remote=16416 dedup_ratio=0.292999 router_tokens_per_expert=[1285, 485, 238, 2, 4, 788, 1425, 104, 7, 1573, 46, 1, 75, 68, 244, 348, 80, 888, 795, 1493, 21, 242, 306, 14, 248, 19, 62, 314, 993, 897, 468, 1137, 368, 223, 710, 364, 173, 1580, 370, 706, 670, 1026, 1261, 58, 1403, 851, 47, 329, 854, 842, 866, 40, 739, 995, 817, 206, 1015, 18, 539, 46, 10, 175, 718, 79] payload_rows_per_dst=[5834, 5660, 6348, 5782, 6144, 5158, 5318, 6090] dup_rows_per_dst=[2828, 2786, 3032, 2494, 2844, 1624, 1678, 1916] combine_rows_per_src=[5834, 5660, 6348, 5782, 6144, 5158, 5318, 6090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56098 payload_rows_remote=48174 dup_rows=9438 dup_rows_remote=8772 dedup_ratio=0.144012 router_tokens_per_expert=[1, 1, 3960, 0, 0, 13, 0, 22, 64, 0, 183, 284, 4023, 7, 644, 0, 1614, 0, 67, 3, 15, 1075, 0, 0, 0, 2, 0, 161, 1826, 3, 0, 2708, 81, 1, 1, 0, 0, 0, 319, 3893, 1390, 4050, 969, 0, 3, 0, 58, 0, 9, 3944, 1, 0, 0, 0, 0, 789, 376, 0, 175, 0, 33, 0, 0, 0] payload_rows_per_dst=[7962, 8192, 4802, 6760, 7924, 4362, 7984, 8112] dup_rows_per_dst=[32, 2218, 746, 2640, 666, 478, 1502, 1156] combine_rows_per_src=[7962, 8192, 4802, 6760, 7924, 4362, 7984, 8112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55760 payload_rows_remote=48598 dup_rows=9776 dup_rows_remote=9272 dedup_ratio=0.149170 router_tokens_per_expert=[0, 8, 3747, 0, 0, 71, 1, 0, 3, 0, 122, 66, 4083, 0, 1, 0, 2828, 0, 727, 2, 0, 8, 243, 62, 0, 12, 0, 0, 3468, 0, 6, 684, 984, 0, 0, 0, 0, 0, 378, 3974, 127, 4023, 1388, 0, 11, 0, 5, 0, 170, 3392, 4, 2, 0, 0, 0, 78, 1376, 0, 671, 1, 42, 0, 0, 0] payload_rows_per_dst=[7626, 8190, 6622, 7162, 8084, 3130, 6938, 8008] dup_rows_per_dst=[28, 360, 1792, 504, 2588, 64, 354, 4086] combine_rows_per_src=[7626, 8190, 6622, 7162, 8084, 3130, 6938, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160325632 dispatch_meta_bytes=216120 combine_payload_bytes=160325632 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160541752 ingress_bytes=235823104 link_bytes=235823104 token_bytes=320867384 weight_movement_bytes=75497472 total_nvlink_bytes=396364856 weight_to_token_ratio=0.235292 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=6912 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.794341343e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166600704 dispatch_meta_bytes=228360 combine_payload_bytes=166600704 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=166829064 ingress_bytes=216932352 link_bytes=216932352 token_bytes=333429768 weight_movement_bytes=50331648 total_nvlink_bytes=383761416 weight_to_token_ratio=0.150951 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.650604547e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53354 payload_rows_remote=45408 dup_rows=12182 dup_rows_remote=7000 dedup_ratio=0.185883 router_tokens_per_expert=[343, 0, 4078, 0, 0, 0, 5, 0, 12, 2, 3, 9, 4095, 0, 105, 0, 523, 0, 1161, 0, 0, 820, 14, 407, 0, 310, 0, 191, 2592, 4, 0, 3639, 138, 0, 0, 0, 1, 1, 132, 4077, 2191, 2897, 534, 1, 78, 0, 0, 2, 661, 3355, 0, 1, 6, 1, 0, 100, 261, 0, 14, 2, 1, 0, 1, 0] payload_rows_per_dst=[8184, 8192, 7946, 5818, 8154, 7392, 7110, 558] dup_rows_per_dst=[668, 260, 5182, 376, 544, 4014, 1138, 0] combine_rows_per_src=[8184, 8192, 7946, 5818, 8154, 7392, 7110, 558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55428 payload_rows_remote=47914 dup_rows=10108 dup_rows_remote=8546 dedup_ratio=0.154236 router_tokens_per_expert=[1, 157, 4028, 1, 0, 1, 0, 0, 1, 1, 7, 107, 4075, 0, 754, 0, 3530, 0, 207, 2, 0, 476, 0, 2, 0, 70, 1, 0, 1190, 0, 8, 3275, 160, 18, 0, 0, 0, 0, 108, 3943, 445, 3263, 1606, 0, 51, 0, 0, 1, 369, 3546, 0, 0, 0, 0, 0, 89, 386, 0, 73, 0, 808, 0, 8, 0] payload_rows_per_dst=[8138, 1730, 7530, 6688, 8002, 8178, 7648, 7514] dup_rows_per_dst=[238, 10, 904, 20, 456, 4178, 2740, 1562] combine_rows_per_src=[8138, 1730, 7530, 6688, 8002, 8178, 7648, 7514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49982 payload_rows_remote=47868 dup_rows=15554 dup_rows_remote=15474 dedup_ratio=0.237335 router_tokens_per_expert=[0, 409, 3820, 0, 0, 0, 2, 0, 1, 3, 0, 1113, 4082, 1, 2, 0, 756, 17, 191, 25, 0, 10, 10, 58, 4, 3, 0, 0, 3564, 0, 126, 3712, 34, 0, 0, 0, 0, 0, 188, 4090, 40, 3838, 980, 0, 68, 0, 9, 0, 75, 2779, 0, 2, 1, 0, 0, 0, 1802, 0, 914, 4, 34, 0, 1, 0] payload_rows_per_dst=[7838, 8170, 1866, 8046, 8186, 2114, 5686, 8076] dup_rows_per_dst=[624, 2234, 268, 6772, 438, 80, 28, 5110] combine_rows_per_src=[7838, 8170, 1866, 8046, 8186, 2114, 5686, 8076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=197320704 dispatch_meta_bytes=227784 combine_payload_bytes=197320704 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=197548488 ingress_bytes=272818176 link_bytes=272818176 token_bytes=394869192 weight_movement_bytes=75497472 total_nvlink_bytes=470366664 weight_to_token_ratio=0.191196 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.075831095e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=199057408 dispatch_meta_bytes=231480 combine_payload_bytes=199057408 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=199288888 ingress_bytes=274554880 link_bytes=274554880 token_bytes=398346296 weight_movement_bytes=75497472 total_nvlink_bytes=473843768 weight_to_token_ratio=0.189527 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.089045406e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50186 payload_rows_remote=42506 dup_rows=15350 dup_rows_remote=14842 dedup_ratio=0.234222 router_tokens_per_expert=[2, 39, 3343, 3, 1, 12, 19, 0, 26, 16, 1, 2000, 3862, 0, 217, 24, 541, 1, 655, 61, 1334, 2116, 28, 62, 0, 109, 189, 45, 820, 79, 55, 543, 85, 77, 6, 70, 1, 42, 66, 3754, 745, 4062, 138, 70, 1558, 2, 0, 0, 8, 3517, 150, 194, 0, 2, 0, 47, 443, 6, 817, 0, 656, 22, 26, 1] payload_rows_per_dst=[7680, 7890, 6996, 5272, 3022, 8190, 7584, 3552] dup_rows_per_dst=[508, 4402, 3686, 908, 244, 4960, 252, 390] combine_rows_per_src=[7680, 7890, 6996, 5272, 3022, 8190, 7584, 3552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=185991168 dispatch_meta_bytes=209632 combine_payload_bytes=185991168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186200800 ingress_bytes=211156992 link_bytes=211156992 token_bytes=372191968 weight_movement_bytes=25165824 total_nvlink_bytes=397357792 weight_to_token_ratio=0.067615 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.606660731e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=196255744 dispatch_meta_bytes=225840 combine_payload_bytes=196255744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=196481584 ingress_bytes=221421568 link_bytes=221421568 token_bytes=392737328 weight_movement_bytes=25165824 total_nvlink_bytes=417903152 weight_to_token_ratio=0.064078 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.684762294e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=196067328 dispatch_meta_bytes=253368 combine_payload_bytes=196067328 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=196320696 ingress_bytes=246398976 link_bytes=246398976 token_bytes=392388024 weight_movement_bytes=50331648 total_nvlink_bytes=442719672 weight_to_token_ratio=0.128270 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.874811509e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=174104576 dispatch_meta_bytes=229392 combine_payload_bytes=174104576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=174333968 ingress_bytes=199270400 link_bytes=199270400 token_bytes=348438544 weight_movement_bytes=25165824 total_nvlink_bytes=373604368 weight_to_token_ratio=0.072225 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.516217500e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62364 payload_rows_remote=54172 dup_rows=3172 dup_rows_remote=2434 dedup_ratio=0.048401 router_tokens_per_expert=[183, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 12, 0, 0, 0, 0, 8, 3963, 6, 4037, 2, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 4085, 0, 0, 387, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2, 3732, 4073, 0, 0, 4082, 0, 17, 0, 0, 129, 0, 3868, 0, 0, 0, 3, 0, 82, 0] payload_rows_per_dst=[8142, 8168, 6170, 8180, 8192, 7466, 8192, 7854] dup_rows_per_dst=[2076, 24, 12, 4, 738, 2, 264, 52] combine_rows_per_src=[8142, 8168, 6170, 8180, 8192, 7466, 8192, 7854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51252 payload_rows_remote=44328 dup_rows=14284 dup_rows_remote=8418 dedup_ratio=0.217957 router_tokens_per_expert=[0, 0, 3986, 0, 0, 0, 0, 0, 73, 0, 0, 82, 3650, 0, 5, 1, 1040, 0, 657, 2, 0, 16, 19, 1, 0, 1, 0, 0, 3726, 0, 68, 3480, 98, 0, 0, 0, 0, 0, 183, 3719, 101, 3998, 1909, 0, 210, 0, 1, 0, 7, 3086, 0, 0, 8, 0, 6, 78, 766, 1, 1719, 0, 71, 0, 0, 0] payload_rows_per_dst=[7972, 2378, 3314, 7630, 7602, 7250, 6924, 8182] dup_rows_per_dst=[0, 102, 156, 500, 638, 2094, 5866, 4928] combine_rows_per_src=[7972, 2378, 3314, 7630, 7602, 7250, 6924, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58300 payload_rows_remote=50108 dup_rows=7236 dup_rows_remote=6158 dedup_ratio=0.110413 router_tokens_per_expert=[1, 0, 125, 0, 226, 0, 0, 0, 0, 4043, 0, 21, 0, 0, 0, 0, 4011, 473, 0, 1543, 38, 0, 0, 0, 0, 0, 1, 0, 194, 402, 0, 4038, 0, 0, 17, 3, 0, 0, 135, 0, 0, 0, 0, 0, 0, 4016, 3976, 0, 0, 4091, 0, 1770, 0, 0, 0, 0, 3619, 3, 0, 0, 0, 0, 22, 0] payload_rows_per_dst=[4372, 8104, 8082, 8192, 8156, 8032, 6108, 7254] dup_rows_per_dst=[364, 24, 3750, 1078, 106, 0, 1880, 34] combine_rows_per_src=[4372, 8104, 8082, 8192, 8156, 8032, 6108, 7254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53902 payload_rows_remote=45810 dup_rows=11634 dup_rows_remote=8518 dedup_ratio=0.177521 router_tokens_per_expert=[0, 1, 3855, 0, 0, 0, 163, 0, 143, 2, 0, 1507, 3919, 19, 13, 1, 2460, 1, 2, 5, 400, 104, 11, 10, 0, 0, 0, 10, 3208, 0, 2, 3870, 798, 0, 0, 0, 2, 0, 41, 3653, 0, 4005, 75, 0, 430, 0, 6, 10, 134, 1680, 16, 1, 0, 1, 42, 17, 920, 0, 977, 0, 241, 0, 13, 0] payload_rows_per_dst=[7818, 8092, 8136, 6438, 7620, 8062, 3750, 3986] dup_rows_per_dst=[220, 3116, 5590, 2, 1368, 990, 32, 316] combine_rows_per_src=[7818, 8092, 8136, 6438, 7620, 8062, 3750, 3986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=221888512 dispatch_meta_bytes=226424 combine_payload_bytes=221888512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=222114936 ingress_bytes=247054336 link_bytes=247054336 token_bytes=444003448 weight_movement_bytes=25165824 total_nvlink_bytes=469169272 weight_to_token_ratio=0.056679 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.879798041e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58840 payload_rows_remote=50784 dup_rows=6696 dup_rows_remote=6484 dedup_ratio=0.102173 router_tokens_per_expert=[10, 0, 145, 0, 80, 1, 0, 0, 0, 4041, 0, 65, 0, 0, 0, 0, 110, 4096, 25, 2280, 0, 0, 0, 0, 0, 0, 2, 0, 114, 366, 0, 4092, 0, 4, 14, 2, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3032, 4031, 0, 1, 3999, 0, 1908, 3, 0, 0, 1, 4080, 70, 0, 0, 0, 0, 154, 6] payload_rows_per_dst=[8192, 8162, 8056, 8188, 8096, 6136, 3824, 8186] dup_rows_per_dst=[5032, 50, 212, 960, 6, 0, 2, 434] combine_rows_per_src=[8192, 8162, 8056, 8188, 8096, 6136, 3824, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52444 payload_rows_remote=44746 dup_rows=13092 dup_rows_remote=12266 dedup_ratio=0.199768 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 5, 0, 0, 4085, 0, 1038, 0, 0, 0, 0, 23, 3434, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3862, 0, 38, 0, 0, 71, 4, 1, 0, 0, 0, 0, 0, 8, 4036, 4080, 0, 0, 4053, 9, 62, 0, 0, 0, 0, 3786, 0, 6, 0, 17, 1, 10, 63] payload_rows_per_dst=[6878, 2076, 8140, 4278, 7112, 8076, 8186, 7698] dup_rows_per_dst=[2, 0, 44, 8, 196, 3784, 8232, 826] combine_rows_per_src=[6878, 2076, 8140, 4278, 7112, 8076, 8186, 7698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56354 payload_rows_remote=48176 dup_rows=9182 dup_rows_remote=9182 dedup_ratio=0.140106 router_tokens_per_expert=[3, 0, 0, 0, 223, 13, 0, 1, 0, 4096, 0, 25, 0, 0, 0, 0, 2724, 406, 2, 924, 13, 0, 0, 0, 0, 0, 1, 0, 15, 534, 1, 3880, 75, 78, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 4089, 4088, 0, 31, 4063, 0, 3801, 0, 3, 0, 0, 3549, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1288, 8192, 7166, 8106, 8192, 8178, 8134, 7098] dup_rows_per_dst=[4, 50, 160, 756, 550, 0, 7662, 0] combine_rows_per_src=[1288, 8192, 7166, 8106, 8192, 8178, 8134, 7098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=181567488 dispatch_meta_bytes=210984 combine_payload_bytes=181567488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=181778472 ingress_bytes=206733312 link_bytes=206733312 token_bytes=363345960 weight_movement_bytes=25165824 total_nvlink_bytes=388511784 weight_to_token_ratio=0.069261 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.573001637e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=205242368 dispatch_meta_bytes=225064 combine_payload_bytes=205242368 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=205467432 ingress_bytes=280739840 link_bytes=280739840 token_bytes=410709800 weight_movement_bytes=75497472 total_nvlink_bytes=486207272 weight_to_token_ratio=0.183822 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=29056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.136105805e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55164 payload_rows_remote=47972 dup_rows=10372 dup_rows_remote=6608 dedup_ratio=0.158264 router_tokens_per_expert=[10, 0, 0, 28, 0, 0, 1, 0, 27, 4054, 0, 828, 0, 0, 0, 55, 45, 3038, 0, 2401, 8, 176, 0, 0, 0, 0, 427, 0, 18, 1, 63, 3959, 0, 0, 0, 0, 0, 0, 42, 0, 1, 220, 41, 0, 4, 3511, 4053, 4, 688, 4096, 1, 32, 7, 27, 95, 0, 2370, 6, 0, 0, 99, 0, 2332, 0] payload_rows_per_dst=[7192, 8164, 8192, 6714, 8152, 7484, 1498, 7768] dup_rows_per_dst=[3764, 3856, 458, 130, 38, 78, 202, 1846] combine_rows_per_src=[7192, 8164, 8192, 6714, 8152, 7484, 1498, 7768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=187637760 dispatch_meta_bytes=217312 combine_payload_bytes=187637760 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=187855072 ingress_bytes=237969408 link_bytes=237969408 token_bytes=375492832 weight_movement_bytes=50331648 total_nvlink_bytes=425824480 weight_to_token_ratio=0.134042 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=25216 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.810672236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=208011264 dispatch_meta_bytes=229072 combine_payload_bytes=208011264 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=208240336 ingress_bytes=283508736 link_bytes=283508736 token_bytes=416251600 weight_movement_bytes=75497472 total_nvlink_bytes=491749072 weight_to_token_ratio=0.181375 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=13952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.157173905e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=183279616 dispatch_meta_bytes=228048 combine_payload_bytes=183279616 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183507664 ingress_bytes=208445440 link_bytes=208445440 token_bytes=366787280 weight_movement_bytes=25165824 total_nvlink_bytes=391953104 weight_to_token_ratio=0.068611 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.586028953e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=197328896 dispatch_meta_bytes=229432 combine_payload_bytes=197328896 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=197558328 ingress_bytes=247660544 link_bytes=247660544 token_bytes=394887224 weight_movement_bytes=50331648 total_nvlink_bytes=445218872 weight_to_token_ratio=0.127458 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.884410584e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=196493312 dispatch_meta_bytes=218320 combine_payload_bytes=196493312 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=196711632 ingress_bytes=246824960 link_bytes=246824960 token_bytes=393204944 weight_movement_bytes=50331648 total_nvlink_bytes=443536592 weight_to_token_ratio=0.128004 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.878052755e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62628 payload_rows_remote=54436 dup_rows=2908 dup_rows_remote=1406 dedup_ratio=0.044373 router_tokens_per_expert=[0, 0, 4077, 0, 0, 412, 3930, 0, 0, 0, 0, 0, 0, 1515, 0, 2276, 3916, 0, 1, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 749, 4079, 0, 0, 2, 0, 0, 4096, 6, 0, 11, 3548, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4071, 69, 0, 0, 1] payload_rows_per_dst=[8154, 7126, 7848, 8160, 8192, 7098, 7862, 8188] dup_rows_per_dst=[824, 456, 0, 0, 1502, 32, 0, 94] combine_rows_per_src=[8154, 7126, 7848, 8160, 8192, 7098, 7862, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61318 payload_rows_remote=53510 dup_rows=4218 dup_rows_remote=4182 dedup_ratio=0.064362 router_tokens_per_expert=[0, 0, 4090, 0, 0, 0, 3790, 2, 0, 0, 121, 0, 0, 2176, 0, 3076, 4045, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 3904, 0, 0, 0, 0, 0, 2557, 0, 0, 167, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 677, 0, 0, 0] payload_rows_per_dst=[8180, 8144, 8166, 7808, 5114, 8134, 7580, 8192] dup_rows_per_dst=[4, 2602, 84, 36, 0, 144, 0, 1348] combine_rows_per_src=[8180, 8144, 8166, 7808, 5114, 8134, 7580, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62242 payload_rows_remote=54078 dup_rows=3294 dup_rows_remote=1632 dedup_ratio=0.050262 router_tokens_per_expert=[11, 0, 1, 0, 226, 0, 4, 0, 0, 3934, 0, 0, 0, 0, 5, 0, 625, 3708, 0, 2898, 4, 0, 0, 0, 0, 0, 1, 0, 0, 536, 0, 3799, 0, 1, 0, 0, 55, 0, 2, 0, 0, 0, 0, 0, 0, 4054, 4050, 0, 12, 4033, 1, 631, 0, 1, 48, 0, 3918, 64, 0, 0, 0, 45, 98, 3] payload_rows_per_dst=[7528, 7504, 6812, 7950, 8180, 8108, 8164, 7996] dup_rows_per_dst=[372, 0, 242, 722, 36, 0, 1662, 260] combine_rows_per_src=[7528, 7504, 6812, 7950, 8180, 8108, 8164, 7996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=222969856 dispatch_meta_bytes=223368 combine_payload_bytes=222969856 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=223193224 ingress_bytes=298467328 link_bytes=298467328 token_bytes=446163080 weight_movement_bytes=75497472 total_nvlink_bytes=521660552 weight_to_token_ratio=0.169215 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.270991506e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57238 payload_rows_remote=49058 dup_rows=8298 dup_rows_remote=5316 dedup_ratio=0.126617 router_tokens_per_expert=[135, 0, 70, 0, 0, 1, 8, 0, 0, 3856, 0, 1513, 0, 0, 0, 212, 19, 3601, 15, 1350, 1, 0, 0, 0, 0, 0, 3, 15, 644, 396, 0, 4077, 0, 5, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3777, 0, 105, 3985, 0, 502, 2, 154, 0, 0, 4081, 0, 1, 1, 0, 0, 32, 0] payload_rows_per_dst=[7512, 8180, 7994, 8180, 7658, 8192, 1358, 8164] dup_rows_per_dst=[2818, 2982, 46, 2090, 128, 0, 168, 66] combine_rows_per_src=[7512, 8180, 7994, 8180, 7658, 8192, 1358, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64122 payload_rows_remote=55930 dup_rows=1414 dup_rows_remote=1320 dedup_ratio=0.021576 router_tokens_per_expert=[7, 0, 4096, 0, 6, 0, 4094, 0, 0, 0, 0, 0, 0, 97, 1, 4016, 4082, 0, 39, 0, 0, 0, 0, 22, 0, 0, 26, 0, 0, 10, 0, 3, 495, 4086, 1, 3, 0, 15, 0, 4096, 0, 0, 11, 3448, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4089, 9, 0, 0, 12] payload_rows_per_dst=[8192, 8082, 8192, 8178, 8192, 6918, 8190, 8178] dup_rows_per_dst=[26, 146, 94, 72, 1028, 0, 6, 42] combine_rows_per_src=[8192, 8082, 8192, 8178, 8192, 6918, 8190, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=219176960 dispatch_meta_bytes=230768 combine_payload_bytes=219176960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=219407728 ingress_bytes=244342784 link_bytes=244342784 token_bytes=438584688 weight_movement_bytes=25165824 total_nvlink_bytes=463750512 weight_to_token_ratio=0.057380 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.859166264e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=221503488 dispatch_meta_bytes=222840 combine_payload_bytes=221503488 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=221726328 ingress_bytes=271835136 link_bytes=271835136 token_bytes=443229816 weight_movement_bytes=50331648 total_nvlink_bytes=493561464 weight_to_token_ratio=0.113557 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.068351297e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61378 payload_rows_remote=53680 dup_rows=4158 dup_rows_remote=4018 dedup_ratio=0.063446 router_tokens_per_expert=[2, 0, 3905, 0, 0, 25, 3586, 15, 0, 0, 1, 0, 0, 118, 0, 4030, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1274, 0, 500, 2, 4030, 132, 18, 0, 0, 0, 4088, 0, 0, 270, 3649, 0, 0, 0, 0, 7, 0, 0, 0, 0, 47, 1, 0, 0, 3, 0, 4084, 4, 0, 0, 44] payload_rows_per_dst=[7842, 8148, 5866, 8192, 8192, 7698, 7268, 8172] dup_rows_per_dst=[52, 150, 0, 3416, 288, 140, 14, 98] combine_rows_per_src=[7842, 8148, 5866, 8192, 8192, 7698, 7268, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50464 payload_rows_remote=42280 dup_rows=15072 dup_rows_remote=15070 dedup_ratio=0.229980 router_tokens_per_expert=[0, 0, 4094, 0, 0, 2508, 4085, 1, 0, 0, 3, 0, 0, 164, 0, 3056, 4041, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 1, 833, 4091, 36, 0, 0, 0, 0, 4088, 0, 0, 0, 1412, 0, 0, 0, 1, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0] payload_rows_per_dst=[5016, 2988, 8114, 1602, 8192, 8188, 8180, 8184] dup_rows_per_dst=[2, 4, 3454, 314, 8466, 2826, 4, 2] combine_rows_per_src=[5016, 2988, 8114, 1602, 8192, 8188, 8180, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51228 payload_rows_remote=43036 dup_rows=14308 dup_rows_remote=3964 dedup_ratio=0.218323 router_tokens_per_expert=[0, 0, 2761, 0, 272, 2504, 3799, 2, 0, 0, 0, 0, 4, 23, 0, 3287, 3934, 12, 3, 8, 111, 85, 0, 0, 639, 0, 15, 42, 3, 2, 0, 0, 2220, 4084, 0, 0, 0, 0, 0, 2808, 0, 0, 0, 297, 0, 0, 0, 0, 139, 0, 0, 0, 0, 60, 2070, 0, 0, 0, 0, 3519, 62, 0, 0, 3] payload_rows_per_dst=[8192, 6636, 8126, 8192, 4488, 594, 8140, 6860] dup_rows_per_dst=[10344, 76, 180, 1378, 176, 0, 2154, 0] combine_rows_per_src=[8192, 6636, 8126, 8192, 4488, 594, 8140, 6860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=200941568 dispatch_meta_bytes=217496 combine_payload_bytes=200941568 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=201159064 ingress_bytes=251273216 link_bytes=251273216 token_bytes=402100632 weight_movement_bytes=50331648 total_nvlink_bytes=452432280 weight_to_token_ratio=0.125172 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.911898844e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=229089280 dispatch_meta_bytes=229000 combine_payload_bytes=229089280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229318280 ingress_bytes=254255104 link_bytes=254255104 token_bytes=458407560 weight_movement_bytes=25165824 total_nvlink_bytes=483573384 weight_to_token_ratio=0.054898 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.934587566e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64810 payload_rows_remote=56618 dup_rows=726 dup_rows_remote=682 dedup_ratio=0.011078 router_tokens_per_expert=[0, 4, 1, 3649, 154, 0, 4042, 0, 52, 0, 5, 0, 48, 0, 0, 0, 4091, 0, 0, 0, 0, 4095, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 18, 0, 1, 0, 1, 0, 3, 0, 3, 4056, 4087, 13, 0, 0, 8, 0, 0, 0, 20, 0, 4093, 0, 0, 0, 0, 246] payload_rows_per_dst=[7608, 8186, 8146, 8182, 8192, 8122, 8188, 8186] dup_rows_per_dst=[8, 108, 0, 0, 44, 6, 28, 532] combine_rows_per_src=[7608, 8186, 8146, 8182, 8192, 8122, 8188, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=219873280 dispatch_meta_bytes=230792 combine_payload_bytes=219873280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=220104072 ingress_bytes=245039104 link_bytes=245039104 token_bytes=439977352 weight_movement_bytes=25165824 total_nvlink_bytes=465143176 weight_to_token_ratio=0.057198 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=24320 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.864464454e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=173178880 dispatch_meta_bytes=229400 combine_payload_bytes=173178880 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=173408280 ingress_bytes=223510528 link_bytes=223510528 token_bytes=346587160 weight_movement_bytes=50331648 total_nvlink_bytes=396918808 weight_to_token_ratio=0.145221 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.700656866e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=176275456 dispatch_meta_bytes=188000 combine_payload_bytes=176275456 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=176463456 ingress_bytes=251772928 link_bytes=251772928 token_bytes=352738912 weight_movement_bytes=75497472 total_nvlink_bytes=428236384 weight_to_token_ratio=0.214032 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.915701075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57512 payload_rows_remote=49320 dup_rows=8024 dup_rows_remote=7960 dedup_ratio=0.122437 router_tokens_per_expert=[0, 0, 0, 4033, 0, 0, 2935, 0, 176, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 4081, 0, 0, 0, 0, 1, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7, 0, 0, 0, 1140, 4096, 0, 7, 0, 0, 0, 3897, 0, 0, 0, 4095, 0, 0, 0, 0, 60] payload_rows_per_dst=[8066, 6234, 8162, 8192, 8182, 2294, 8192, 8190] dup_rows_per_dst=[0, 28, 0, 64, 4, 0, 7808, 120] combine_rows_per_src=[8066, 6234, 8162, 8192, 8182, 2294, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=231907328 dispatch_meta_bytes=229200 combine_payload_bytes=231907328 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=232136528 ingress_bytes=282238976 link_bytes=282238976 token_bytes=464043856 weight_movement_bytes=50331648 total_nvlink_bytes=514375504 weight_to_token_ratio=0.108463 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.147512498e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55044 payload_rows_remote=47142 dup_rows=10492 dup_rows_remote=10358 dedup_ratio=0.160095 router_tokens_per_expert=[0, 0, 4090, 0, 0, 3, 3945, 0, 0, 11, 0, 0, 0, 3091, 0, 1968, 3949, 0, 75, 0, 1, 18, 0, 0, 15, 0, 3, 20, 0, 3, 0, 253, 228, 3736, 1, 0, 0, 0, 0, 3133, 0, 0, 45, 4077, 31, 0, 1, 0, 1, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 2, 5, 0, 1] payload_rows_per_dst=[526, 8170, 8026, 7878, 6360, 8192, 7902, 7990] dup_rows_per_dst=[2, 1970, 60, 182, 364, 7774, 134, 6] combine_rows_per_src=[526, 8170, 8026, 7878, 6360, 8192, 7902, 7990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60454 payload_rows_remote=52414 dup_rows=5082 dup_rows_remote=2284 dedup_ratio=0.077545 router_tokens_per_expert=[3, 1, 3932, 0, 0, 14, 4050, 29, 0, 142, 0, 1, 0, 3477, 2, 1797, 3661, 0, 2, 0, 0, 0, 0, 0, 1, 1, 104, 1, 0, 862, 0, 1, 167, 4094, 15, 0, 0, 0, 0, 2420, 0, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4090, 0, 0, 0, 0] payload_rows_per_dst=[7940, 8040, 7322, 8192, 4880, 7798, 8100, 8182] dup_rows_per_dst=[240, 2798, 4, 1936, 102, 0, 2, 0] combine_rows_per_src=[7940, 8040, 7322, 8192, 4880, 7798, 8100, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=202014720 dispatch_meta_bytes=229120 combine_payload_bytes=202014720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202243840 ingress_bytes=227180544 link_bytes=227180544 token_bytes=404258560 weight_movement_bytes=25165824 total_nvlink_bytes=429424384 weight_to_token_ratio=0.062252 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=30336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.728581447e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60220 payload_rows_remote=52268 dup_rows=5316 dup_rows_remote=5316 dedup_ratio=0.081116 router_tokens_per_expert=[0, 0, 0, 4035, 0, 0, 4067, 0, 0, 0, 0, 0, 1273, 0, 0, 3, 4096, 0, 1, 0, 0, 4071, 3975, 0, 0, 0, 0, 64, 20, 0, 0, 0, 0, 1, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 2, 1588, 4093, 0, 0, 0, 0, 0, 151, 0, 0, 0, 4094, 0, 0, 0, 0, 1164] payload_rows_per_dst=[8070, 8192, 7952, 8192, 8176, 4166, 7284, 8188] dup_rows_per_dst=[0, 2494, 0, 168, 42, 92, 192, 2328] combine_rows_per_src=[8070, 8192, 7952, 8192, 8176, 4166, 7284, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63028 payload_rows_remote=56188 dup_rows=2508 dup_rows_remote=2476 dedup_ratio=0.038269 router_tokens_per_expert=[0, 0, 0, 4006, 189, 3, 3932, 1, 260, 0, 0, 0, 0, 0, 0, 0, 4090, 13, 0, 0, 0, 4095, 4073, 0, 0, 0, 0, 0, 40, 0, 0, 2, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 334, 0, 0, 0, 9, 3093, 3816, 0, 0, 0, 2, 0, 1220, 30, 0, 0, 3543, 0, 0, 12, 0, 0] payload_rows_per_dst=[8168, 8178, 8170, 8180, 8190, 6840, 8192, 7110] dup_rows_per_dst=[230, 206, 2, 84, 10, 32, 1944, 0] combine_rows_per_src=[8168, 8178, 8170, 8180, 8190, 6840, 8192, 7110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=193093632 dispatch_meta_bytes=230000 combine_payload_bytes=193093632 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193323632 ingress_bytes=218259456 link_bytes=218259456 token_bytes=386417264 weight_movement_bytes=25165824 total_nvlink_bytes=411583088 weight_to_token_ratio=0.065126 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.660702275e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60590 payload_rows_remote=52468 dup_rows=4946 dup_rows_remote=2930 dedup_ratio=0.075470 router_tokens_per_expert=[0, 0, 0, 4088, 0, 10, 4091, 0, 12, 0, 0, 0, 711, 0, 0, 0, 4038, 0, 0, 0, 2, 4054, 3935, 8, 44, 0, 0, 403, 27, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 96, 2, 1, 0, 0, 3048, 4096, 0, 6, 3, 0, 7, 7, 0, 0, 0, 4057, 0, 0, 4, 0, 4] payload_rows_per_dst=[7518, 8184, 5970, 8192, 8108, 6304, 8192, 8122] dup_rows_per_dst=[8, 1444, 0, 836, 582, 14, 46, 2016] combine_rows_per_src=[7518, 8184, 5970, 8192, 8108, 6304, 8192, 8122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52420 payload_rows_remote=45282 dup_rows=13116 dup_rows_remote=13096 dedup_ratio=0.200134 router_tokens_per_expert=[2, 0, 2, 0, 14, 0, 3561, 0, 427, 0, 0, 290, 3205, 0, 0, 322, 3856, 0, 4, 0, 2, 4096, 3660, 14, 177, 0, 6, 7, 522, 217, 0, 114, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 13, 5, 5, 0, 0, 3987, 3736, 0, 0, 2, 0, 433, 62, 0, 16, 10, 3432, 120, 0, 444, 0, 0] payload_rows_per_dst=[7138, 7692, 7742, 4772, 8192, 8046, 4722, 4116] dup_rows_per_dst=[20, 796, 3852, 792, 10, 7446, 52, 148] combine_rows_per_src=[7138, 7692, 7742, 4772, 8192, 8046, 4722, 4116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=214687744 dispatch_meta_bytes=218792 combine_payload_bytes=214687744 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=214906536 ingress_bytes=265019392 link_bytes=265019392 token_bytes=429594280 weight_movement_bytes=50331648 total_nvlink_bytes=479925928 weight_to_token_ratio=0.117161 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.016491360e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=230336 combine_payload_bytes=214089728 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=214320064 ingress_bytes=339918848 link_bytes=339918848 token_bytes=428409792 weight_movement_bytes=125829120 total_nvlink_bytes=554238912 weight_to_token_ratio=0.293712 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.586389679e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=230146048 dispatch_meta_bytes=234656 combine_payload_bytes=230146048 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230380704 ingress_bytes=255311872 link_bytes=255311872 token_bytes=460526752 weight_movement_bytes=25165824 total_nvlink_bytes=485692576 weight_to_token_ratio=0.054646 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.942628349e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63974 payload_rows_remote=56114 dup_rows=1562 dup_rows_remote=1562 dedup_ratio=0.023834 router_tokens_per_expert=[0, 0, 4093, 4071, 0, 3936, 0, 0, 0, 3, 0, 0, 0, 4066, 2, 0, 0, 65, 0, 1, 0, 0, 4091, 0, 3231, 0, 444, 0, 0, 0, 0, 689, 0, 0, 0, 0, 3928, 0, 2, 4081, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 41, 10, 7, 0, 0, 0, 0] payload_rows_per_dst=[7872, 8142, 8182, 7370, 7860, 8166, 8192, 8190] dup_rows_per_dst=[0, 842, 132, 516, 0, 0, 4, 68] combine_rows_per_src=[7872, 8142, 8182, 7370, 7860, 8166, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=214908928 dispatch_meta_bytes=221592 combine_payload_bytes=214908928 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=215130520 ingress_bytes=265240576 link_bytes=265240576 token_bytes=430039448 weight_movement_bytes=50331648 total_nvlink_bytes=480371096 weight_to_token_ratio=0.117040 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=29696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.018174315e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=185475072 dispatch_meta_bytes=233512 combine_payload_bytes=185475072 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=185708584 ingress_bytes=235806720 link_bytes=235806720 token_bytes=371183656 weight_movement_bytes=50331648 total_nvlink_bytes=421515304 weight_to_token_ratio=0.135598 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.794216680e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57922 payload_rows_remote=57178 dup_rows=7614 dup_rows_remote=7614 dedup_ratio=0.116180 router_tokens_per_expert=[0, 0, 4096, 4096, 0, 4089, 0, 0, 0, 0, 0, 58, 0, 59, 0, 0, 0, 0, 0, 68, 0, 0, 3992, 45, 3933, 7, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 4087, 0, 46, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 34, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8152, 8092, 744, 8176, 8192, 8192, 8192] dup_rows_per_dst=[7132, 200, 118, 0, 90, 0, 0, 74] combine_rows_per_src=[8182, 8152, 8092, 744, 8176, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=229842944 dispatch_meta_bytes=230704 combine_payload_bytes=229842944 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=230073648 ingress_bytes=280174592 link_bytes=280174592 token_bytes=459916592 weight_movement_bytes=50331648 total_nvlink_bytes=510248240 weight_to_token_ratio=0.109436 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.131804921e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62004 payload_rows_remote=53828 dup_rows=3532 dup_rows_remote=1612 dedup_ratio=0.053894 router_tokens_per_expert=[10, 1, 12, 3386, 5, 0, 4079, 0, 512, 0, 0, 0, 0, 0, 0, 1, 4029, 0, 0, 1, 0, 3983, 3675, 0, 0, 0, 0, 31, 23, 0, 0, 242, 0, 0, 0, 0, 1, 0, 1, 12, 0, 0, 15, 0, 1, 8, 4, 3564, 3801, 0, 0, 0, 2, 46, 1199, 0, 0, 0, 4074, 0, 0, 18, 0, 32] payload_rows_per_dst=[6820, 8164, 7352, 8192, 7994, 7128, 8176, 8178] dup_rows_per_dst=[8, 1020, 0, 458, 0, 56, 1920, 70] combine_rows_per_src=[6820, 8164, 7352, 8192, 7994, 7128, 8176, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=234201088 dispatch_meta_bytes=259168 combine_payload_bytes=234201088 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=234460256 ingress_bytes=309698560 link_bytes=309698560 token_bytes=468661344 weight_movement_bytes=75497472 total_nvlink_bytes=544158816 weight_to_token_ratio=0.161092 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=12032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.356448205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53832 payload_rows_remote=49338 dup_rows=11704 dup_rows_remote=11588 dedup_ratio=0.178589 router_tokens_per_expert=[2, 0, 0, 4092, 3, 1, 4048, 22, 0, 0, 0, 1, 73, 0, 0, 0, 4073, 0, 4, 0, 0, 3891, 4089, 1, 0, 0, 0, 4, 0, 0, 0, 539, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 2, 3625, 3867, 0, 0, 0, 0, 0, 300, 0, 5, 0, 4095, 1, 0, 0, 0, 15] payload_rows_per_dst=[8192, 4494, 8180, 8188, 7802, 8184, 600, 8192] dup_rows_per_dst=[3682, 116, 8, 1044, 8, 6806, 0, 40] combine_rows_per_src=[8192, 4494, 8180, 8188, 7802, 8184, 600, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57244 payload_rows_remote=49052 dup_rows=8292 dup_rows_remote=5828 dedup_ratio=0.126526 router_tokens_per_expert=[0, 0, 4096, 4084, 0, 298, 1, 0, 3, 0, 0, 55, 0, 4048, 0, 0, 0, 82, 0, 0, 0, 0, 4094, 1152, 4068, 10, 20, 0, 0, 0, 0, 2502, 5, 0, 0, 0, 4082, 0, 0, 4090, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 3, 0, 0, 2, 0] payload_rows_per_dst=[8140, 8112, 8192, 60, 8170, 8192, 8192, 8186] dup_rows_per_dst=[594, 5104, 2464, 0, 4, 118, 8, 0] combine_rows_per_src=[8140, 8112, 8192, 60, 8170, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57832 payload_rows_remote=49766 dup_rows=7704 dup_rows_remote=7702 dedup_ratio=0.117554 router_tokens_per_expert=[0, 18, 4096, 3989, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 3327, 44, 0, 7, 2475, 81, 448, 0, 0, 3712, 38, 0, 123, 1077, 0, 0, 0, 0, 3870, 0, 0, 0, 0, 3993, 0, 61, 3903, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[2748, 6742, 8024, 8066, 8014, 8066, 8192, 7980] dup_rows_per_dst=[36, 0, 5498, 2074, 94, 2, 0, 0] combine_rows_per_src=[2748, 6742, 8024, 8066, 8014, 8066, 8192, 7980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=220479488 dispatch_meta_bytes=221760 combine_payload_bytes=220479488 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=220701248 ingress_bytes=270811136 link_bytes=270811136 token_bytes=441180736 weight_movement_bytes=50331648 total_nvlink_bytes=491512384 weight_to_token_ratio=0.114084 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.060559840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55700 payload_rows_remote=50322 dup_rows=9836 dup_rows_remote=9780 dedup_ratio=0.150085 router_tokens_per_expert=[0, 0, 4007, 4064, 0, 0, 27, 0, 69, 142, 0, 0, 0, 3922, 206, 0, 0, 0, 0, 0, 0, 0, 2023, 194, 10, 0, 3160, 0, 4, 0, 0, 4003, 0, 0, 30, 99, 3801, 0, 0, 3928, 4, 0, 192, 0, 0, 0, 0, 0, 1, 0, 0, 7, 34, 0, 0, 0, 6, 2803, 32, 0, 0, 0, 0, 0] payload_rows_per_dst=[5378, 8188, 4150, 6320, 7882, 8172, 8048, 7562] dup_rows_per_dst=[56, 8496, 284, 8, 34, 20, 50, 888] combine_rows_per_src=[5378, 8188, 4150, 6320, 7882, 8172, 8048, 7562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48254 payload_rows_remote=40232 dup_rows=17282 dup_rows_remote=17282 dedup_ratio=0.263702 router_tokens_per_expert=[0, 5, 4087, 4009, 0, 1959, 0, 0, 0, 1, 0, 0, 26, 671, 0, 0, 0, 0, 0, 3, 0, 0, 4049, 67, 1269, 1122, 2915, 0, 0, 278, 0, 4053, 14, 8, 0, 19, 4079, 13, 0, 4096, 12, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 1, 0, 0, 0] payload_rows_per_dst=[3812, 1396, 2274, 8180, 8192, 8192, 8186, 8022] dup_rows_per_dst=[0, 0, 94, 16062, 976, 26, 124, 0] combine_rows_per_src=[3812, 1396, 2274, 8180, 8192, 8192, 8186, 8022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=202088448 dispatch_meta_bytes=243704 combine_payload_bytes=202088448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202332152 ingress_bytes=227254272 link_bytes=227254272 token_bytes=404420600 weight_movement_bytes=25165824 total_nvlink_bytes=429586424 weight_to_token_ratio=0.062227 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.729142432e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60022 payload_rows_remote=51832 dup_rows=5514 dup_rows_remote=1218 dedup_ratio=0.084137 router_tokens_per_expert=[0, 2314, 179, 0, 0, 35, 5, 0, 0, 0, 0, 0, 3, 0, 4088, 0, 3, 0, 0, 0, 1, 0, 1, 0, 0, 4086, 0, 0, 0, 0, 4062, 6, 0, 0, 1, 0, 537, 0, 0, 1611, 0, 0, 4096, 0, 4093, 0, 2, 0, 0, 0, 4094, 22, 2, 2477, 786, 0, 256, 0, 0, 0, 1, 0, 0, 7] payload_rows_per_dst=[4892, 8176, 8192, 8124, 8190, 8192, 6072, 8184] dup_rows_per_dst=[174, 6, 4, 12, 4296, 4, 502, 516] combine_rows_per_src=[4892, 8176, 8192, 8124, 8190, 8192, 6072, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=203841536 dispatch_meta_bytes=229872 combine_payload_bytes=203841536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=204071408 ingress_bytes=229007360 link_bytes=229007360 token_bytes=407912944 weight_movement_bytes=25165824 total_nvlink_bytes=433078768 weight_to_token_ratio=0.061694 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.742481406e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49392 payload_rows_remote=48126 dup_rows=16144 dup_rows_remote=16114 dedup_ratio=0.246338 router_tokens_per_expert=[0, 2012, 114, 0, 0, 1211, 0, 0, 0, 0, 0, 0, 449, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 155, 0, 0, 0, 0, 199, 0, 0, 161, 0, 0, 4096, 0, 4096, 0, 3646, 0, 0, 0, 4049, 0, 0, 35, 246, 0, 4095, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6610, 8188, 8192, 1266, 8192, 8192, 562, 8190] dup_rows_per_dst=[64, 898, 0, 30, 626, 7292, 0, 7234] combine_rows_per_src=[6610, 8188, 8192, 1266, 8192, 8192, 562, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=206118912 dispatch_meta_bytes=240408 combine_payload_bytes=206118912 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=206359320 ingress_bytes=306782208 link_bytes=306782208 token_bytes=412478232 weight_movement_bytes=100663296 total_nvlink_bytes=513141528 weight_to_token_ratio=0.244045 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19840 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.334258136e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=200916992 dispatch_meta_bytes=219520 combine_payload_bytes=200916992 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=201136512 ingress_bytes=251248640 link_bytes=251248640 token_bytes=402053504 weight_movement_bytes=50331648 total_nvlink_bytes=452385152 weight_to_token_ratio=0.125186 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27008 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.911711849e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=164790272 dispatch_meta_bytes=230056 combine_payload_bytes=164790272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=165020328 ingress_bytes=189956096 link_bytes=189956096 token_bytes=329810600 weight_movement_bytes=25165824 total_nvlink_bytes=354976424 weight_to_token_ratio=0.076304 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.445346408e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=212303872 dispatch_meta_bytes=212200 combine_payload_bytes=212303872 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=212516072 ingress_bytes=237469696 link_bytes=237469696 token_bytes=424819944 weight_movement_bytes=25165824 total_nvlink_bytes=449985768 weight_to_token_ratio=0.059239 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.806870005e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=197124096 dispatch_meta_bytes=256960 combine_payload_bytes=197124096 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=197381056 ingress_bytes=247455744 link_bytes=247455744 token_bytes=394505152 weight_movement_bytes=50331648 total_nvlink_bytes=444836800 weight_to_token_ratio=0.127582 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.882852293e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58610 payload_rows_remote=50450 dup_rows=6926 dup_rows_remote=6926 dedup_ratio=0.105682 router_tokens_per_expert=[0, 0, 4072, 4007, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 1029, 13, 0, 0, 3, 128, 0, 0, 0, 3882, 10, 2600, 10, 930, 0, 2, 140, 0, 3762, 0, 0, 0, 0, 3933, 0, 0, 4047, 39, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 26, 0, 2, 0, 1] payload_rows_per_dst=[8134, 2064, 7896, 8188, 7866, 8172, 8160, 8130] dup_rows_per_dst=[0, 20, 150, 6700, 0, 2, 0, 54] combine_rows_per_src=[8134, 2064, 7896, 8188, 7866, 8172, 8160, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58302 payload_rows_remote=50236 dup_rows=7234 dup_rows_remote=7222 dedup_ratio=0.110382 router_tokens_per_expert=[0, 1, 4094, 4087, 0, 4083, 0, 0, 0, 1, 0, 0, 0, 4032, 0, 0, 0, 494, 1, 1, 0, 0, 4068, 429, 2642, 261, 334, 0, 3, 26, 0, 6, 0, 0, 0, 22, 4066, 0, 1, 4095, 0, 3, 0, 1, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1] payload_rows_per_dst=[8172, 8066, 8168, 1152, 8176, 8192, 8188, 8188] dup_rows_per_dst=[5280, 12, 1818, 96, 2, 24, 0, 2] combine_rows_per_src=[8172, 8066, 8168, 1152, 8176, 8192, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50502 payload_rows_remote=42310 dup_rows=15034 dup_rows_remote=8948 dedup_ratio=0.229401 router_tokens_per_expert=[0, 3579, 364, 0, 516, 0, 0, 0, 0, 10, 0, 0, 214, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3019, 6, 0, 0, 3, 39, 0, 3, 0, 0, 0, 259, 0, 0, 652, 0, 0, 4082, 0, 4083, 0, 3019, 38, 4, 0, 3580, 71, 0, 1186, 1, 0, 3920, 0, 0, 0, 0, 0, 0, 24] payload_rows_per_dst=[7786, 8192, 8166, 90, 5064, 8192, 5098, 7914] dup_rows_per_dst=[1132, 448, 0, 6, 418, 6086, 932, 6012] combine_rows_per_src=[7786, 8192, 8166, 90, 5064, 8192, 5098, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=206643200 dispatch_meta_bytes=229504 combine_payload_bytes=206643200 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=206872704 ingress_bytes=256974848 link_bytes=256974848 token_bytes=413515904 weight_movement_bytes=50331648 total_nvlink_bytes=463847552 weight_to_token_ratio=0.121716 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.955281675e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43008 payload_rows_remote=34944 dup_rows=22528 dup_rows_remote=15278 dedup_ratio=0.343750 router_tokens_per_expert=[0, 3435, 302, 0, 8, 3885, 27, 0, 0, 0, 0, 0, 1934, 237, 3570, 0, 2, 0, 0, 0, 0, 0, 1629, 0, 0, 1112, 0, 17, 0, 0, 1252, 2081, 8, 224, 0, 0, 0, 0, 431, 223, 0, 0, 3627, 0, 4093, 0, 29, 0, 436, 0, 3618, 241, 0, 0, 145, 0, 183, 0, 0, 0, 15, 1, 0, 3] payload_rows_per_dst=[8064, 8048, 8190, 6334, 8106, 2728, 1168, 370] dup_rows_per_dst=[7250, 8018, 3258, 2590, 902, 0, 476, 34] combine_rows_per_src=[8064, 8048, 8190, 6334, 8106, 2728, 1168, 370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54096 payload_rows_remote=45924 dup_rows=11440 dup_rows_remote=11440 dedup_ratio=0.174561 router_tokens_per_expert=[0, 4024, 1844, 0, 0, 10, 0, 0, 0, 10, 0, 0, 1, 0, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 2, 3345, 0, 0, 81, 6, 0, 196, 0, 0, 256, 101, 2, 4077, 0, 4086, 0, 21, 0, 52, 0, 4090, 7, 0, 2338, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 7210, 8172, 8108, 8192, 8160, 4754, 1312] dup_rows_per_dst=[3568, 0, 0, 6524, 1066, 242, 40, 0] combine_rows_per_src=[8188, 7210, 8172, 8108, 8192, 8160, 4754, 1312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=40342 payload_rows_remote=32166 dup_rows=25194 dup_rows_remote=17872 dedup_ratio=0.384430 router_tokens_per_expert=[0, 3899, 990, 0, 7, 0, 86, 0, 0, 3, 0, 0, 42, 0, 4018, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 2, 0, 2861, 0, 0, 0, 0, 0, 15, 0, 0, 847, 0, 0, 4047, 0, 4013, 1, 69, 0, 60, 0, 4044, 0, 0, 5, 0, 0, 3664, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5988, 86, 4596, 8050, 1702, 3568, 8176, 8176] dup_rows_per_dst=[1776, 24, 0, 5712, 22, 2202, 8136, 7322] combine_rows_per_src=[5988, 86, 4596, 8050, 1702, 3568, 8176, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=205766656 dispatch_meta_bytes=229832 combine_payload_bytes=205766656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=205996488 ingress_bytes=230932480 link_bytes=230932480 token_bytes=411763144 weight_movement_bytes=25165824 total_nvlink_bytes=436928968 weight_to_token_ratio=0.061117 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.757129345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51364 payload_rows_remote=43172 dup_rows=14172 dup_rows_remote=13992 dedup_ratio=0.216248 router_tokens_per_expert=[0, 0, 4094, 37, 1132, 0, 661, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1414, 0, 0, 0, 1785, 0, 0, 0, 0, 94, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4037, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 4082, 0, 3090] payload_rows_per_dst=[78, 2254, 8190, 8192, 8192, 8076, 8192, 8190] dup_rows_per_dst=[6, 0, 6396, 180, 0, 0, 114, 7476] combine_rows_per_src=[78, 2254, 8190, 8192, 8192, 8076, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55778 payload_rows_remote=47586 dup_rows=9758 dup_rows_remote=9748 dedup_ratio=0.148895 router_tokens_per_expert=[0, 0, 4004, 37, 4094, 0, 88, 4096, 0, 1486, 0, 1, 0, 0, 0, 0, 0, 2, 35, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 4094, 0, 4094, 5, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 2921, 0, 1, 0, 0, 0, 3181, 0, 417] payload_rows_per_dst=[74, 8190, 8076, 8192, 8192, 8180, 8192, 6682] dup_rows_per_dst=[0, 2972, 6, 226, 10, 10, 5842, 692] combine_rows_per_src=[74, 8190, 8076, 8192, 8192, 8180, 8192, 6682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=173301760 dispatch_meta_bytes=205032 combine_payload_bytes=173301760 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=173506792 ingress_bytes=273965056 link_bytes=273965056 token_bytes=346808552 weight_movement_bytes=100663296 total_nvlink_bytes=447471848 weight_to_token_ratio=0.290256 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.084557527e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=143130624 dispatch_meta_bytes=200888 combine_payload_bytes=143130624 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=143331512 ingress_bytes=168296448 link_bytes=168296448 token_bytes=286462136 weight_movement_bytes=25165824 total_nvlink_bytes=311627960 weight_to_token_ratio=0.087850 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.280541513e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=188104704 dispatch_meta_bytes=229456 combine_payload_bytes=188104704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188334160 ingress_bytes=213270528 link_bytes=213270528 token_bytes=376438864 weight_movement_bytes=25165824 total_nvlink_bytes=401604688 weight_to_token_ratio=0.066852 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.622742298e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=131751936 dispatch_meta_bytes=200152 combine_payload_bytes=131751936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=131952088 ingress_bytes=156917760 link_bytes=156917760 token_bytes=263704024 weight_movement_bytes=25165824 total_nvlink_bytes=288869848 weight_to_token_ratio=0.095432 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.193962845e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=176832512 dispatch_meta_bytes=228656 combine_payload_bytes=176832512 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=177061168 ingress_bytes=201998336 link_bytes=201998336 token_bytes=353893680 weight_movement_bytes=25165824 total_nvlink_bytes=379059504 weight_to_token_ratio=0.071111 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.536973941e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=194912256 dispatch_meta_bytes=229336 combine_payload_bytes=194912256 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=195141592 ingress_bytes=245243904 link_bytes=245243904 token_bytes=390053848 weight_movement_bytes=50331648 total_nvlink_bytes=440385496 weight_to_token_ratio=0.129038 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.866022746e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47680 payload_rows_remote=39606 dup_rows=17856 dup_rows_remote=4210 dedup_ratio=0.272461 router_tokens_per_expert=[11, 2801, 0, 0, 0, 267, 0, 2, 0, 0, 0, 0, 12, 3, 4081, 36, 48, 0, 0, 13, 0, 0, 185, 0, 0, 3446, 0, 6, 0, 0, 75, 1583, 0, 0, 0, 0, 0, 0, 0, 2533, 0, 1, 4085, 0, 4013, 6, 982, 0, 0, 0, 4037, 0, 0, 2952, 0, 0, 1589, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[6120, 5886, 8192, 5116, 5066, 1990, 8074, 7236] dup_rows_per_dst=[42, 74, 326, 516, 0, 416, 13646, 2836] combine_rows_per_src=[6120, 5886, 8192, 5116, 5066, 1990, 8074, 7236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51812 payload_rows_remote=43640 dup_rows=13724 dup_rows_remote=11614 dedup_ratio=0.209412 router_tokens_per_expert=[0, 1898, 28, 0, 6, 9, 0, 0, 0, 0, 0, 0, 1103, 0, 4034, 4, 0, 1, 0, 0, 6, 0, 0, 21, 0, 4065, 0, 0, 0, 0, 2463, 1, 0, 0, 0, 0, 1, 0, 0, 2759, 1, 2, 4095, 0, 4096, 4, 34, 0, 970, 0, 4087, 6, 1, 0, 2973, 0, 96, 0, 0, 1, 0, 2, 1, 0] payload_rows_per_dst=[3874, 8172, 8192, 8162, 8178, 8190, 6850, 194] dup_rows_per_dst=[8, 2110, 56, 4896, 5516, 82, 1050, 6] combine_rows_per_src=[3874, 8172, 8192, 8162, 8178, 8190, 6850, 194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=162226176 dispatch_meta_bytes=175264 combine_payload_bytes=162226176 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=162401440 ingress_bytes=212557824 link_bytes=212557824 token_bytes=324627616 weight_movement_bytes=50331648 total_nvlink_bytes=374959264 weight_to_token_ratio=0.155044 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=9856 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.617319444e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=40756 payload_rows_remote=32564 dup_rows=24780 dup_rows_remote=4574 dedup_ratio=0.378113 router_tokens_per_expert=[18, 135, 4008, 0, 4095, 0, 866, 981, 0, 3957, 0, 6, 0, 0, 0, 0, 0, 0, 0, 262, 2, 0, 68, 0, 0, 0, 0, 228, 0, 0, 3633, 0, 4096, 2046, 0, 1, 0, 0, 307, 1, 0, 0, 0, 0, 2, 0, 2931, 3917, 0, 0, 30, 0, 2, 0, 0, 130, 0, 0, 0, 0, 18, 476, 1, 551] payload_rows_per_dst=[8192, 7926, 548, 7632, 6118, 7956, 320, 2064] dup_rows_per_dst=[20206, 0, 116, 90, 2356, 1980, 4, 28] combine_rows_per_src=[8192, 7926, 548, 7632, 6118, 7956, 320, 2064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59436 payload_rows_remote=51254 dup_rows=6100 dup_rows_remote=6092 dedup_ratio=0.093079 router_tokens_per_expert=[0, 0, 4081, 104, 3577, 0, 35, 4096, 0, 4072, 0, 3, 0, 0, 0, 0, 0, 0, 2, 11, 0, 0, 1, 0, 0, 0, 0, 617, 0, 0, 4087, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 3, 0, 0, 0, 0, 1708, 0, 0, 0, 0, 17, 1905, 0, 309] payload_rows_per_dst=[7738, 8144, 8182, 8192, 8192, 8084, 6466, 4438] dup_rows_per_dst=[2994, 6, 8, 1216, 4, 0, 1778, 94] combine_rows_per_src=[7738, 8144, 8182, 8192, 8192, 8084, 6466, 4438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=178749440 dispatch_meta_bytes=221016 combine_payload_bytes=178749440 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=178970456 ingress_bytes=229081088 link_bytes=229081088 token_bytes=357719896 weight_movement_bytes=50331648 total_nvlink_bytes=408051544 weight_to_token_ratio=0.140701 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=26496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.743042391e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55034 payload_rows_remote=48346 dup_rows=10502 dup_rows_remote=10502 dedup_ratio=0.160248 router_tokens_per_expert=[0, 1, 3645, 491, 16, 0, 1895, 4010, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 1, 0, 0, 8, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 3269, 0, 0, 0, 0, 0, 0, 234, 3492, 0, 0, 1, 0, 0, 249, 0, 1681] payload_rows_per_dst=[984, 8146, 8166, 8190, 8192, 6688, 8192, 6476] dup_rows_per_dst=[0, 32, 1998, 16, 0, 0, 7280, 1176] combine_rows_per_src=[984, 8146, 8166, 8190, 8192, 6688, 8192, 6476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55952 payload_rows_remote=47818 dup_rows=9584 dup_rows_remote=4014 dedup_ratio=0.146240 router_tokens_per_expert=[0, 0, 4093, 0, 4094, 0, 3826, 4033, 0, 381, 9, 0, 0, 0, 16, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 3, 82, 3, 0, 4074, 0, 4094, 12, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 139, 3956, 0, 0, 103, 0, 0, 0, 0, 820, 7, 0, 1, 0, 0, 2262, 0, 756] payload_rows_per_dst=[0, 8192, 8190, 6902, 8188, 8154, 8192, 8134] dup_rows_per_dst=[0, 810, 0, 144, 26, 1314, 1720, 5570] combine_rows_per_src=[0, 8192, 8190, 6902, 8188, 8154, 8192, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59760 payload_rows_remote=51638 dup_rows=5776 dup_rows_remote=5776 dedup_ratio=0.088135 router_tokens_per_expert=[0, 0, 0, 1242, 498, 0, 0, 0, 0, 5, 0, 0, 269, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 0, 0, 0, 2956, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 159, 7, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 3399, 0, 0, 3790] payload_rows_per_dst=[3388, 8192, 8192, 8122, 7796, 8192, 8188, 7690] dup_rows_per_dst=[92, 548, 0, 0, 4262, 0, 332, 542] combine_rows_per_src=[3388, 8192, 8192, 8122, 7796, 8192, 8188, 7690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60348 payload_rows_remote=57092 dup_rows=5188 dup_rows_remote=5188 dedup_ratio=0.079163 router_tokens_per_expert=[0, 0, 0, 3979, 0, 0, 2, 0, 0, 0, 0, 0, 51, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4085, 0, 0, 0, 0, 1628, 4093, 5, 0, 0, 0, 0, 0, 0, 2, 16, 12, 0, 3, 0, 4085, 0, 0, 0, 1, 0, 4095, 0, 0, 2514] payload_rows_per_dst=[7962, 8192, 8192, 8186, 3256, 8190, 8178, 8192] dup_rows_per_dst=[0, 80, 16, 0, 0, 6, 58, 5028] combine_rows_per_src=[7962, 8192, 8192, 8186, 3256, 8190, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=133382144 dispatch_meta_bytes=148552 combine_payload_bytes=133382144 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=133530696 ingress_bytes=259211264 link_bytes=259211264 token_bytes=266912840 weight_movement_bytes=125829120 total_nvlink_bytes=392741960 weight_to_token_ratio=0.471424 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11520 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.972298217e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=209936384 dispatch_meta_bytes=229384 combine_payload_bytes=209936384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210165768 ingress_bytes=235102208 link_bytes=235102208 token_bytes=420102152 weight_movement_bytes=25165824 total_nvlink_bytes=445267976 weight_to_token_ratio=0.059904 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27520 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788856157e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=195862528 dispatch_meta_bytes=207328 combine_payload_bytes=195862528 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=196069856 ingress_bytes=221028352 link_bytes=221028352 token_bytes=391932384 weight_movement_bytes=25165824 total_nvlink_bytes=417098208 weight_to_token_ratio=0.064210 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.681770375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=198025216 dispatch_meta_bytes=235392 combine_payload_bytes=198025216 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=198260608 ingress_bytes=248356864 link_bytes=248356864 token_bytes=396285824 weight_movement_bytes=50331648 total_nvlink_bytes=446617472 weight_to_token_ratio=0.127008 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.889708775e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=233848832 dispatch_meta_bytes=249120 combine_payload_bytes=233848832 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=234097952 ingress_bytes=309346304 link_bytes=309346304 token_bytes=467946784 weight_movement_bytes=75497472 total_nvlink_bytes=543444256 weight_to_token_ratio=0.161338 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21504 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.353767944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211509248 dispatch_meta_bytes=229656 combine_payload_bytes=211509248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=211738904 ingress_bytes=236675072 link_bytes=236675072 token_bytes=423248152 weight_movement_bytes=25165824 total_nvlink_bytes=448413976 weight_to_token_ratio=0.059459 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.800823835e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55198 payload_rows_remote=47058 dup_rows=10338 dup_rows_remote=9912 dedup_ratio=0.157745 router_tokens_per_expert=[0, 37, 3892, 0, 4006, 0, 1712, 4055, 0, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4094, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 3120, 0, 0, 1, 0, 0, 0, 15, 212, 0, 0, 0, 0, 0, 2904, 1, 1149] payload_rows_per_dst=[74, 8184, 7788, 8162, 8188, 6568, 8140, 8094] dup_rows_per_dst=[0, 6140, 0, 0, 334, 0, 426, 3438] combine_rows_per_src=[74, 8184, 7788, 8162, 8188, 6568, 8140, 8094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46908 payload_rows_remote=42368 dup_rows=18628 dup_rows_remote=18628 dedup_ratio=0.284241 router_tokens_per_expert=[0, 0, 4085, 69, 3725, 0, 3637, 4073, 3, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 326, 0, 0, 0, 0, 64, 0, 0, 4037, 0, 4092, 450, 0, 11, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 247, 4007, 0, 9, 0, 1, 1, 0, 0, 328, 0, 1, 0, 0, 0, 814, 0, 504] payload_rows_per_dst=[8192, 4540, 7250, 8166, 7258, 8158, 658, 2686] dup_rows_per_dst=[16998, 0, 622, 36, 286, 368, 20, 298] combine_rows_per_src=[8192, 4540, 7250, 8166, 7258, 8158, 658, 2686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=173539328 dispatch_meta_bytes=243984 combine_payload_bytes=173539328 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=173783312 ingress_bytes=198705152 link_bytes=198705152 token_bytes=347322640 weight_movement_bytes=25165824 total_nvlink_bytes=372488464 weight_to_token_ratio=0.072457 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.511916616e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=192749568 dispatch_meta_bytes=227880 combine_payload_bytes=192749568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=192977448 ingress_bytes=217915392 link_bytes=217915392 token_bytes=385727016 weight_movement_bytes=25165824 total_nvlink_bytes=410892840 weight_to_token_ratio=0.065243 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.658084346e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50604 payload_rows_remote=42412 dup_rows=14932 dup_rows_remote=6848 dedup_ratio=0.227844 router_tokens_per_expert=[0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 7, 0, 63, 0, 0, 0, 3939, 0, 7, 0, 0, 0, 0, 83, 0, 0, 3092, 0, 282, 0, 0, 0, 4032, 238, 3831, 0, 0, 0, 0, 1521, 4089, 0, 0, 27, 0, 0, 0, 0, 0, 92, 3, 77, 0, 0, 3210, 0, 7, 26, 0, 0, 4096, 0, 0, 4] payload_rows_per_dst=[8192, 7788, 8048, 8190, 3502, 8178, 6632, 74] dup_rows_per_dst=[8084, 14, 10, 6622, 16, 54, 132, 0] combine_rows_per_src=[8192, 7788, 8048, 8190, 3502, 8178, 6632, 74] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57122 payload_rows_remote=48940 dup_rows=8414 dup_rows_remote=8310 dedup_ratio=0.128387 router_tokens_per_expert=[0, 0, 0, 4091, 22, 0, 0, 0, 0, 185, 0, 0, 61, 0, 0, 0, 4082, 0, 0, 0, 0, 61, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4013, 0, 4088, 0, 0, 0, 0, 3787, 4096, 0, 0, 1, 0, 16, 0, 0, 0, 136, 193, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 3876, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8182, 8032, 8140, 8192, 8192, 0] dup_rows_per_dst=[34, 476, 104, 0, 7186, 34, 580, 0] combine_rows_per_src=[8192, 8192, 8182, 8032, 8140, 8192, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62056 payload_rows_remote=53864 dup_rows=3480 dup_rows_remote=3394 dedup_ratio=0.053101 router_tokens_per_expert=[0, 0, 0, 1725, 2341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 60, 4076, 0, 0, 0, 0, 3636, 4096, 0, 32, 0, 0, 11, 0, 0, 25, 0, 0, 0, 0, 0, 3390, 28, 0, 0, 0, 0, 4018, 0, 0, 1167] payload_rows_per_dst=[7082, 8152, 8192, 8134, 7276, 8192, 6840, 8188] dup_rows_per_dst=[1050, 0, 0, 0, 116, 86, 46, 2182] combine_rows_per_src=[7082, 8152, 8192, 8134, 7276, 8192, 6840, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53340 payload_rows_remote=45148 dup_rows=12196 dup_rows_remote=1400 dedup_ratio=0.186096 router_tokens_per_expert=[0, 0, 0, 4057, 1, 0, 0, 0, 0, 18, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 15, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4085, 0, 0, 0, 0, 260, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 86, 0, 0, 0, 0, 4050, 13, 4, 0, 0, 1, 4093, 0, 0, 3801] payload_rows_per_dst=[8114, 7400, 8152, 2150, 6030, 8022, 5280, 8192] dup_rows_per_dst=[2, 28, 48, 0, 1270, 0, 52, 10796] combine_rows_per_src=[8114, 7400, 8152, 2150, 6030, 8022, 5280, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64682 payload_rows_remote=56504 dup_rows=854 dup_rows_remote=854 dedup_ratio=0.013031 router_tokens_per_expert=[6, 1, 4095, 2, 0, 2, 0, 0, 4044, 0, 18, 20, 0, 3692, 0, 4075, 0, 0, 0, 0, 4096, 4094, 4, 0, 67, 0, 0, 1, 0, 5, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 6, 351, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7450, 8190, 8192, 8178, 8192, 8164, 8124] dup_rows_per_dst=[20, 10, 6, 146, 0, 6, 0, 666] combine_rows_per_src=[8192, 7450, 8190, 8192, 8178, 8192, 8164, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64500 payload_rows_remote=56308 dup_rows=1036 dup_rows_remote=1026 dedup_ratio=0.015808 router_tokens_per_expert=[1, 0, 4086, 0, 0, 0, 0, 0, 3726, 0, 0, 41, 0, 4022, 0, 3894, 368, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 102, 0, 0, 0, 135, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8046, 8192, 8192, 8192, 8192, 8054, 7460] dup_rows_per_dst=[2, 80, 736, 10, 0, 204, 4, 0] combine_rows_per_src=[8172, 8046, 8192, 8192, 8192, 8192, 8054, 7460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=173719552 dispatch_meta_bytes=197040 combine_payload_bytes=173719552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=173916592 ingress_bytes=198885376 link_bytes=198885376 token_bytes=347636144 weight_movement_bytes=25165824 total_nvlink_bytes=372801968 weight_to_token_ratio=0.072391 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20736 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.513287912e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=200458240 dispatch_meta_bytes=229000 combine_payload_bytes=200458240 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=200687240 ingress_bytes=225624064 link_bytes=225624064 token_bytes=401145480 weight_movement_bytes=25165824 total_nvlink_bytes=426311304 weight_to_token_ratio=0.062735 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.716738433e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=184926208 dispatch_meta_bytes=186192 combine_payload_bytes=184926208 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=185112400 ingress_bytes=260423680 link_bytes=260423680 token_bytes=370038608 weight_movement_bytes=75497472 total_nvlink_bytes=445536080 weight_to_token_ratio=0.204026 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.981523302e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=220626944 dispatch_meta_bytes=229032 combine_payload_bytes=220626944 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=220855976 ingress_bytes=270958592 link_bytes=270958592 token_bytes=441482920 weight_movement_bytes=50331648 total_nvlink_bytes=491814568 weight_to_token_ratio=0.114006 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.061681810e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55804 payload_rows_remote=47616 dup_rows=9732 dup_rows_remote=9728 dedup_ratio=0.148499 router_tokens_per_expert=[0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 4088, 0, 0, 7, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3953, 18, 4088, 5, 0, 0, 0, 4063, 4091, 1, 0, 61, 0, 2, 0, 0, 0, 3606, 626, 0, 0, 0, 3835, 0, 1, 0, 0, 0, 4088, 0, 1, 4] payload_rows_per_dst=[6998, 8188, 8192, 7908, 8128, 8192, 8188, 10] dup_rows_per_dst=[186, 4, 12, 0, 1464, 118, 7946, 2] combine_rows_per_src=[6998, 8188, 8192, 7908, 8128, 8192, 8188, 10] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=231440384 dispatch_meta_bytes=229432 combine_payload_bytes=231440384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=231669816 ingress_bytes=256606208 link_bytes=256606208 token_bytes=463110200 weight_movement_bytes=25165824 total_nvlink_bytes=488276024 weight_to_token_ratio=0.054341 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.952476751e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230637568 dispatch_meta_bytes=229336 combine_payload_bytes=230637568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230866904 ingress_bytes=255803392 link_bytes=255803392 token_bytes=461504472 weight_movement_bytes=25165824 total_nvlink_bytes=486670296 weight_to_token_ratio=0.054530 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.946368249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58320 payload_rows_remote=53460 dup_rows=7216 dup_rows_remote=7184 dedup_ratio=0.110107 router_tokens_per_expert=[0, 0, 0, 1333, 1743, 0, 4, 0, 0, 0, 2, 0, 44, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 18, 0, 0, 3529, 4040, 0, 0, 0, 0, 0, 0, 0, 1, 128, 0, 0, 0, 1, 2316, 0, 0, 0, 0, 0, 3965, 0, 0, 3355] payload_rows_per_dst=[5522, 8190, 8192, 8192, 7094, 8080, 4860, 8190] dup_rows_per_dst=[638, 92, 4, 0, 0, 0, 32, 6450] combine_rows_per_src=[5522, 8190, 8192, 8192, 7094, 8080, 4860, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=195035136 dispatch_meta_bytes=229376 combine_payload_bytes=195035136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=195264512 ingress_bytes=220200960 link_bytes=220200960 token_bytes=390299648 weight_movement_bytes=25165824 total_nvlink_bytes=415465472 weight_to_token_ratio=0.064478 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.675474877e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=218972160 dispatch_meta_bytes=242576 combine_payload_bytes=218972160 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=219214736 ingress_bytes=269303808 link_bytes=269303808 token_bytes=438186896 weight_movement_bytes=50331648 total_nvlink_bytes=488518544 weight_to_token_ratio=0.114863 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.049090816e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54130 payload_rows_remote=45938 dup_rows=11406 dup_rows_remote=6360 dedup_ratio=0.174042 router_tokens_per_expert=[0, 0, 3577, 0, 186, 0, 0, 54, 4077, 0, 0, 371, 14, 3620, 0, 3928, 0, 0, 0, 0, 4096, 4095, 0, 0, 224, 0, 60, 0, 0, 103, 0, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4095, 0, 127, 0, 0, 0, 0, 946, 0, 0, 0, 16, 28, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8104, 8192, 6722, 6280, 276, 8190, 8174] dup_rows_per_dst=[5046, 3342, 2164, 78, 0, 0, 776, 0] combine_rows_per_src=[8192, 8104, 8192, 6722, 6280, 276, 8190, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60250 payload_rows_remote=52058 dup_rows=5286 dup_rows_remote=5112 dedup_ratio=0.080658 router_tokens_per_expert=[0, 0, 3982, 0, 0, 0, 0, 0, 4081, 0, 430, 0, 2, 3407, 0, 4086, 87, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 295, 0, 0, 0, 5, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7964, 7710, 8192, 8192, 8192, 3658, 8178, 8164] dup_rows_per_dst=[0, 4610, 174, 6, 8, 482, 6, 0] combine_rows_per_src=[7964, 7710, 8192, 8192, 8192, 3658, 8178, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54478 payload_rows_remote=46304 dup_rows=11058 dup_rows_remote=11056 dedup_ratio=0.168732 router_tokens_per_expert=[0, 0, 4096, 2, 3, 0, 0, 0, 4087, 0, 98, 0, 21, 310, 0, 4084, 0, 0, 0, 0, 4083, 3903, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4043, 3, 3907, 0, 0, 0, 13, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 754, 4652, 8170, 8176, 8186, 8174, 8174] dup_rows_per_dst=[10, 104, 0, 2, 2088, 8814, 38, 2] combine_rows_per_src=[8192, 754, 4652, 8170, 8176, 8186, 8174, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57072 payload_rows_remote=48880 dup_rows=8464 dup_rows_remote=1926 dedup_ratio=0.129150 router_tokens_per_expert=[0, 12, 3496, 0, 0, 0, 0, 0, 3926, 0, 20, 0, 0, 677, 0, 4031, 155, 0, 0, 1, 4096, 4022, 243, 0, 64, 0, 0, 0, 0, 53, 499, 0, 4029, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 48, 4096, 0, 3221, 0, 6, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7016, 1394, 8178, 8192, 8064, 8192, 8184, 7852] dup_rows_per_dst=[0, 0, 664, 1232, 0, 6538, 30, 0] combine_rows_per_src=[7016, 1394, 8178, 8192, 8064, 8192, 8184, 7852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61144 payload_rows_remote=55304 dup_rows=4392 dup_rows_remote=4284 dedup_ratio=0.067017 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3813, 0, 1, 0, 0, 0, 2, 4096, 0, 4, 5, 4078, 4093, 0, 1442, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 56, 0, 2918, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 10, 6, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 3, 4090] payload_rows_per_dst=[8188, 7628, 8192, 8192, 5840, 8170, 6848, 8086] dup_rows_per_dst=[0, 0, 22, 2886, 108, 12, 20, 1344] combine_rows_per_src=[8188, 7628, 8192, 8192, 5840, 8170, 6848, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60092 payload_rows_remote=51986 dup_rows=5444 dup_rows_remote=5412 dedup_ratio=0.083069 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 77, 2351, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 16, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1798, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 6492, 8192, 8106, 8192, 8192, 8192, 4534] dup_rows_per_dst=[2, 1802, 0, 32, 10, 2, 3596, 0] combine_rows_per_src=[8192, 6492, 8192, 8106, 8192, 8192, 8192, 4534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=188162048 dispatch_meta_bytes=209192 combine_payload_bytes=188162048 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188371240 ingress_bytes=213327872 link_bytes=213327872 token_bytes=376533288 weight_movement_bytes=25165824 total_nvlink_bytes=401699112 weight_to_token_ratio=0.066836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.623178619e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=213229568 dispatch_meta_bytes=228680 combine_payload_bytes=213229568 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=213458248 ingress_bytes=288727040 link_bytes=288727040 token_bytes=426687816 weight_movement_bytes=75497472 total_nvlink_bytes=502185288 weight_to_token_ratio=0.176938 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.196879169e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=200212480 dispatch_meta_bytes=203224 combine_payload_bytes=200212480 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=200415704 ingress_bytes=275709952 link_bytes=275709952 token_bytes=400628184 weight_movement_bytes=75497472 total_nvlink_bytes=476125656 weight_to_token_ratio=0.188448 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.097834169e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57996 payload_rows_remote=49804 dup_rows=7540 dup_rows_remote=80 dedup_ratio=0.115051 router_tokens_per_expert=[0, 0, 2268, 0, 0, 2, 0, 0, 4095, 0, 93, 0, 0, 3655, 1, 4091, 0, 13, 0, 6, 4095, 4088, 12, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4082, 0, 0, 0, 0, 1, 0, 0, 0, 4, 1, 0, 4077, 0, 2165, 0, 1, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4540, 8192, 8192, 8192, 8166, 4338, 8186, 8190] dup_rows_per_dst=[0, 7460, 46, 14, 0, 2, 18, 0] combine_rows_per_src=[4540, 8192, 8192, 8192, 8166, 4338, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=189661184 dispatch_meta_bytes=229440 combine_payload_bytes=189661184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189890624 ingress_bytes=214827008 link_bytes=214827008 token_bytes=379551808 weight_movement_bytes=25165824 total_nvlink_bytes=404717632 weight_to_token_ratio=0.066304 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.634585312e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=226525184 dispatch_meta_bytes=238352 combine_payload_bytes=226525184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=226763536 ingress_bytes=251691008 link_bytes=251691008 token_bytes=453288720 weight_movement_bytes=25165824 total_nvlink_bytes=478454544 weight_to_token_ratio=0.055518 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.915077758e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64490 payload_rows_remote=56986 dup_rows=1046 dup_rows_remote=1016 dedup_ratio=0.015961 router_tokens_per_expert=[0, 1, 4087, 0, 0, 0, 0, 7, 4078, 0, 0, 67, 17, 4056, 0, 3717, 3, 0, 0, 0, 4096, 3976, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4077, 0, 147, 0, 41, 0, 0, 0, 0, 0, 4, 5, 287, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8116, 7958, 8192, 8192, 8162, 7504, 8184] dup_rows_per_dst=[8, 164, 0, 4, 0, 294, 30, 546] combine_rows_per_src=[8182, 8116, 7958, 8192, 8192, 8162, 7504, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=212934656 dispatch_meta_bytes=229592 combine_payload_bytes=212934656 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=213164248 ingress_bytes=288432128 link_bytes=288432128 token_bytes=426098904 weight_movement_bytes=75497472 total_nvlink_bytes=501596376 weight_to_token_ratio=0.177183 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.194635229e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=203997184 dispatch_meta_bytes=199536 combine_payload_bytes=203997184 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=204196720 ingress_bytes=279494656 link_bytes=279494656 token_bytes=408193904 weight_movement_bytes=75497472 total_nvlink_bytes=483691376 weight_to_token_ratio=0.184955 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=23552 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.126631394e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233414656 dispatch_meta_bytes=232008 combine_payload_bytes=233414656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233646664 ingress_bytes=258580480 link_bytes=258580480 token_bytes=467061320 weight_movement_bytes=25165824 total_nvlink_bytes=492227144 weight_to_token_ratio=0.053881 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.967498680e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52514 payload_rows_remote=44322 dup_rows=13022 dup_rows_remote=12414 dedup_ratio=0.198700 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 381, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3989, 4017, 0, 3669, 53, 0, 0, 4082, 0, 0, 0, 0, 0, 3132, 0, 2578, 0, 0, 2, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 0, 3840] payload_rows_per_dst=[8192, 5448, 8184, 8192, 8186, 6630, 7682, 0] dup_rows_per_dst=[608, 0, 1740, 7416, 3238, 0, 20, 0] combine_rows_per_src=[8192, 5448, 8184, 8192, 8186, 6630, 7682, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56736 payload_rows_remote=48544 dup_rows=8800 dup_rows_remote=8798 dedup_ratio=0.134277 router_tokens_per_expert=[0, 0, 6, 0, 0, 0, 61, 0, 18, 0, 1527, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4079, 4080, 0, 696, 93, 0, 0, 4096, 0, 0, 0, 0, 0, 1195, 0, 3688, 0, 0, 8, 0, 0, 0, 0, 7, 0, 0, 0, 9, 0, 1005, 3, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 4090] payload_rows_per_dst=[8192, 8050, 8192, 8192, 7746, 8172, 8192, 0] dup_rows_per_dst=[102, 3060, 2, 1578, 2036, 0, 2022, 0] combine_rows_per_src=[8192, 8050, 8192, 8192, 7746, 8172, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56922 payload_rows_remote=48730 dup_rows=8614 dup_rows_remote=8584 dedup_ratio=0.131439 router_tokens_per_expert=[0, 0, 9, 0, 0, 0, 7, 0, 0, 0, 4082, 2, 0, 7, 0, 0, 0, 3848, 0, 0, 0, 4096, 4093, 0, 115, 0, 0, 0, 4070, 0, 23, 0, 0, 0, 1433, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 2366, 166, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8186, 8178, 7696, 8184, 3230, 8192, 5064, 8192] dup_rows_per_dst=[32, 4, 0, 232, 316, 30, 0, 8000] combine_rows_per_src=[8186, 8178, 7696, 8184, 3230, 8192, 5064, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50072 payload_rows_remote=41880 dup_rows=15464 dup_rows_remote=3072 dedup_ratio=0.235962 router_tokens_per_expert=[0, 0, 7, 0, 22, 5, 77, 0, 0, 4, 1083, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4056, 4096, 0, 65, 10, 0, 0, 4096, 1, 0, 0, 0, 0, 22, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 0, 0, 0, 4095] payload_rows_per_dst=[7116, 2174, 7132, 2234, 8158, 8112, 6954, 8192] dup_rows_per_dst=[222, 0, 0, 16, 2832, 2, 0, 12392] combine_rows_per_src=[7116, 2174, 7132, 2234, 8158, 8112, 6954, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65358 payload_rows_remote=57192 dup_rows=178 dup_rows_remote=178 dedup_ratio=0.002716 router_tokens_per_expert=[0, 0, 0, 0, 146, 1, 3913, 0, 2, 0, 6, 0, 0, 0, 74, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4093, 4083, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 3] payload_rows_per_dst=[8116, 8192, 8180, 8184, 8166, 8186, 8192, 8142] dup_rows_per_dst=[4, 160, 0, 0, 0, 0, 8, 6] combine_rows_per_src=[8116, 8192, 8180, 8184, 8166, 8186, 8192, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64728 payload_rows_remote=56536 dup_rows=808 dup_rows_remote=804 dedup_ratio=0.012329 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3729, 8, 0, 0, 77, 0, 0, 0, 66, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0] payload_rows_per_dst=[7474, 8192, 8192, 8192, 8192, 8192, 8192, 8102] dup_rows_per_dst=[0, 286, 0, 4, 518, 0, 0, 0] combine_rows_per_src=[7474, 8192, 8192, 8192, 8192, 8192, 8192, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=181542912 dispatch_meta_bytes=226944 combine_payload_bytes=181542912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=181769856 ingress_bytes=206708736 link_bytes=206708736 token_bytes=363312768 weight_movement_bytes=25165824 total_nvlink_bytes=388478592 weight_to_token_ratio=0.069268 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.572814642e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=198836224 dispatch_meta_bytes=229368 combine_payload_bytes=198836224 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=199065592 ingress_bytes=274333696 link_bytes=274333696 token_bytes=397901816 weight_movement_bytes=75497472 total_nvlink_bytes=473399288 weight_to_token_ratio=0.189739 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.087362451e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53000 payload_rows_remote=44808 dup_rows=12536 dup_rows_remote=4306 dedup_ratio=0.191284 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 33, 1, 4096, 1, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 4083, 4095, 0, 47, 0, 3, 0, 4091, 0, 4, 0, 0, 0, 1250, 0, 709, 0, 0, 0, 0, 0, 0, 1, 700, 2, 0, 0, 0, 4, 1398, 0, 0, 1, 14, 0, 0, 0, 4080, 0, 0, 1, 0, 4087] payload_rows_per_dst=[8190, 8192, 8134, 8190, 3918, 8188, 8186, 2] dup_rows_per_dst=[0, 8230, 0, 100, 0, 1384, 2822, 0] combine_rows_per_src=[8190, 8192, 8134, 8190, 3918, 8188, 8186, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=199598080 dispatch_meta_bytes=229256 combine_payload_bytes=199598080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199827336 ingress_bytes=224763904 link_bytes=224763904 token_bytes=399425416 weight_movement_bytes=25165824 total_nvlink_bytes=424591240 weight_to_token_ratio=0.063005 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.710193609e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=171540480 dispatch_meta_bytes=179808 combine_payload_bytes=171540480 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=171720288 ingress_bytes=247037952 link_bytes=247037952 token_bytes=343260768 weight_movement_bytes=75497472 total_nvlink_bytes=418758240 weight_to_token_ratio=0.219942 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=17280 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.879673378e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54272 payload_rows_remote=48336 dup_rows=11264 dup_rows_remote=11262 dedup_ratio=0.171875 router_tokens_per_expert=[0, 0, 0, 0, 24, 0, 78, 0, 0, 0, 3139, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 4057, 4055, 0, 6, 0, 0, 0, 4087, 0, 3, 0, 0, 0, 13, 28, 2719, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 1, 2968, 0, 0, 0, 0, 0, 0, 0, 3303, 4, 0, 0, 0, 4090] payload_rows_per_dst=[8192, 6278, 8050, 1986, 7478, 8168, 5936, 8184] dup_rows_per_dst=[122, 0, 0, 0, 4248, 282, 2, 6610] combine_rows_per_src=[8192, 6278, 8050, 1986, 7478, 8168, 5936, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234258432 dispatch_meta_bytes=229480 combine_payload_bytes=234258432 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234487912 ingress_bytes=259424256 link_bytes=259424256 token_bytes=468746344 weight_movement_bytes=25165824 total_nvlink_bytes=493912168 weight_to_token_ratio=0.053688 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.973918840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=231571456 dispatch_meta_bytes=229360 combine_payload_bytes=231571456 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=231800816 ingress_bytes=281903104 link_bytes=281903104 token_bytes=463372272 weight_movement_bytes=50331648 total_nvlink_bytes=513703920 weight_to_token_ratio=0.108620 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.144956900e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=183533568 dispatch_meta_bytes=196456 combine_payload_bytes=183533568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183730024 ingress_bytes=208699392 link_bytes=208699392 token_bytes=367263592 weight_movement_bytes=25165824 total_nvlink_bytes=392429416 weight_to_token_ratio=0.068523 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=20224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.587961234e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=197984256 dispatch_meta_bytes=238392 combine_payload_bytes=197984256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198222648 ingress_bytes=223150080 link_bytes=223150080 token_bytes=396206904 weight_movement_bytes=25165824 total_nvlink_bytes=421372728 weight_to_token_ratio=0.063517 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.697914273e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57096 payload_rows_remote=48904 dup_rows=8440 dup_rows_remote=4372 dedup_ratio=0.128784 router_tokens_per_expert=[0, 0, 0, 0, 2042, 0, 4088, 0, 0, 0, 237, 0, 35, 0, 0, 49, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 4034, 352, 1837, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4033, 3216, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3602, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 2950, 8192, 7382, 6792, 7204] dup_rows_per_dst=[4068, 518, 38, 0, 3490, 326, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 2950, 8192, 7382, 6792, 7204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64388 payload_rows_remote=56196 dup_rows=1148 dup_rows_remote=1148 dedup_ratio=0.017517 router_tokens_per_expert=[6, 0, 0, 0, 289, 4, 3719, 0, 0, 1, 352, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 52, 4, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4085, 4068, 4096, 0, 0, 0, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0, 3761, 0, 0] payload_rows_per_dst=[7776, 8192, 8192, 8192, 8192, 8136, 8186, 7522] dup_rows_per_dst=[260, 760, 0, 0, 122, 0, 6, 0] combine_rows_per_src=[7776, 8192, 8192, 8192, 8192, 8136, 8186, 7522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62742 payload_rows_remote=54750 dup_rows=2794 dup_rows_remote=2794 dedup_ratio=0.042633 router_tokens_per_expert=[0, 0, 0, 0, 1161, 63, 3592, 1, 0, 0, 6, 0, 0, 0, 51, 0, 0, 6, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3996, 3830, 0, 0, 0, 0, 0, 0, 1, 619, 0, 0, 0, 0, 0, 3593, 0, 0] payload_rows_per_dst=[8174, 8192, 8078, 7268, 7660, 7992, 8192, 7186] dup_rows_per_dst=[1460, 114, 0, 0, 0, 0, 1220, 0] combine_rows_per_src=[8174, 8192, 8078, 7268, 7660, 7992, 8192, 7186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50798 payload_rows_remote=42736 dup_rows=14738 dup_rows_remote=9652 dedup_ratio=0.224884 router_tokens_per_expert=[0, 0, 0, 0, 362, 0, 3725, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 4094, 4074, 0, 70, 0, 0, 2, 0, 0, 0, 0, 14, 0, 0, 0, 4094, 4078, 4094, 1, 0, 0, 34, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4019, 0, 3] payload_rows_per_dst=[5242, 1652, 8190, 8188, 4372, 6900, 8192, 8062] dup_rows_per_dst=[10, 0, 76, 7308, 140, 2046, 72, 5086] combine_rows_per_src=[5242, 1652, 8190, 8188, 4372, 6900, 8192, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=200310784 dispatch_meta_bytes=213104 combine_payload_bytes=200310784 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=200523888 ingress_bytes=275808256 link_bytes=275808256 token_bytes=400834672 weight_movement_bytes=75497472 total_nvlink_bytes=476332144 weight_to_token_ratio=0.188351 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.098582149e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=230178816 dispatch_meta_bytes=229376 combine_payload_bytes=230178816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230408192 ingress_bytes=255344640 link_bytes=255344640 token_bytes=460587008 weight_movement_bytes=25165824 total_nvlink_bytes=485752832 weight_to_token_ratio=0.054639 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.942877676e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60148 payload_rows_remote=51962 dup_rows=5388 dup_rows_remote=5388 dedup_ratio=0.082214 router_tokens_per_expert=[3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 14, 0, 0, 3, 0, 0, 3714, 3, 4093, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 1, 0, 4095, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 4086, 0] payload_rows_per_dst=[6530, 8192, 7456, 5228, 8186, 8192, 8190, 8174] dup_rows_per_dst=[3694, 4, 6, 1680, 0, 2, 2, 0] combine_rows_per_src=[6530, 8192, 7456, 5228, 8186, 8192, 8190, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64628 payload_rows_remote=56444 dup_rows=908 dup_rows_remote=908 dedup_ratio=0.013855 router_tokens_per_expert=[436, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3722, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4019, 0] payload_rows_per_dst=[8192, 8192, 7444, 8184, 8192, 8192, 8192, 8040] dup_rows_per_dst=[872, 4, 2, 0, 0, 2, 28, 0] combine_rows_per_src=[8192, 8192, 7444, 8184, 8192, 8192, 8192, 8040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63512 payload_rows_remote=55324 dup_rows=2024 dup_rows_remote=160 dedup_ratio=0.030884 router_tokens_per_expert=[0, 0, 0, 0, 26, 0, 3707, 0, 0, 0, 908, 2, 1, 0, 23, 0, 0, 0, 0, 4095, 0, 5, 0, 3, 0, 0, 9, 0, 58, 1, 3417, 4092, 11, 37, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4092, 4090, 0, 1, 1, 2, 1, 0, 0, 14, 0, 1, 0, 24, 0, 4065, 0, 0] payload_rows_per_dst=[7464, 8188, 8190, 6950, 8190, 8184, 8172, 8174] dup_rows_per_dst=[2, 1864, 16, 20, 92, 0, 24, 6] combine_rows_per_src=[7464, 8188, 8190, 6950, 8190, 8184, 8172, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=224256000 dispatch_meta_bytes=230176 combine_payload_bytes=224256000 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=224486176 ingress_bytes=324919296 link_bytes=324919296 token_bytes=448742176 weight_movement_bytes=100663296 total_nvlink_bytes=549405472 weight_to_token_ratio=0.224323 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.055147 gemm_active_groups=6 gemm_peak_group_rows=29184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.472260419e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=175046656 dispatch_meta_bytes=209552 combine_payload_bytes=175046656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=175256208 ingress_bytes=200212480 link_bytes=200212480 token_bytes=350302864 weight_movement_bytes=25165824 total_nvlink_bytes=375468688 weight_to_token_ratio=0.071840 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.523385640e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62476 payload_rows_remote=54284 dup_rows=3060 dup_rows_remote=2964 dedup_ratio=0.046692 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 3644, 0, 0, 0, 1119, 0, 0, 0, 343, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 37, 0, 9, 0, 0, 0, 4094, 4055, 0, 137, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4058, 4096, 3967, 0, 0, 0, 45, 0, 0, 2, 39, 0, 0, 0, 0, 0, 3023, 0, 0] payload_rows_per_dst=[7292, 8180, 8192, 8192, 8190, 8192, 8192, 6046] dup_rows_per_dst=[2, 2854, 0, 88, 20, 0, 96, 0] combine_rows_per_src=[7292, 8180, 8192, 8192, 8190, 8192, 8192, 6046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=212836352 dispatch_meta_bytes=229400 combine_payload_bytes=212836352 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=213065752 ingress_bytes=263168000 link_bytes=263168000 token_bytes=425902104 weight_movement_bytes=50331648 total_nvlink_bytes=476233752 weight_to_token_ratio=0.118177 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.002404406e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=231194624 dispatch_meta_bytes=229408 combine_payload_bytes=231194624 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=231424032 ingress_bytes=306692096 link_bytes=306692096 token_bytes=462618656 weight_movement_bytes=75497472 total_nvlink_bytes=538116128 weight_to_token_ratio=0.163196 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.333572487e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=226607104 dispatch_meta_bytes=221936 combine_payload_bytes=226607104 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=226829040 ingress_bytes=251772928 link_bytes=251772928 token_bytes=453436144 weight_movement_bytes=25165824 total_nvlink_bytes=478601968 weight_to_token_ratio=0.055500 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.915701075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=222347264 dispatch_meta_bytes=228992 combine_payload_bytes=222347264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=222576256 ingress_bytes=247513088 link_bytes=247513088 token_bytes=444923520 weight_movement_bytes=25165824 total_nvlink_bytes=470089344 weight_to_token_ratio=0.056562 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.883288614e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52492 payload_rows_remote=44312 dup_rows=13044 dup_rows_remote=7394 dedup_ratio=0.199036 router_tokens_per_expert=[4089, 0, 0, 0, 0, 0, 10, 0, 0, 29, 0, 0, 0, 0, 0, 0, 3, 63, 0, 0, 0, 0, 0, 1961, 0, 2816, 0, 0, 0, 0, 96, 3965, 0, 1292, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 4078, 0, 0, 3986, 0, 0, 1, 0, 0, 34, 0, 0, 0, 4096, 215, 0, 0, 0, 1924, 0] payload_rows_per_dst=[8180, 8192, 8192, 258, 8192, 7392, 7810, 4276] dup_rows_per_dst=[5650, 58, 3792, 0, 3542, 0, 0, 2] combine_rows_per_src=[8180, 8192, 8192, 258, 8192, 7392, 7810, 4276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64006 payload_rows_remote=55814 dup_rows=1530 dup_rows_remote=1500 dedup_ratio=0.023346 router_tokens_per_expert=[555, 2, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 3508, 0, 1, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 120, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3923, 0] payload_rows_per_dst=[8188, 8192, 8192, 7016, 8186, 8192, 8192, 7848] dup_rows_per_dst=[1114, 140, 30, 0, 6, 0, 240, 0] combine_rows_per_src=[8188, 8192, 8192, 7016, 8186, 8192, 8192, 7848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59330 payload_rows_remote=51140 dup_rows=6206 dup_rows_remote=6206 dedup_ratio=0.094696 router_tokens_per_expert=[179, 0, 0, 1372, 0, 0, 0, 28, 0, 309, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3777, 0, 4096, 0, 0, 0, 0, 0, 2526, 0, 266, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4068, 1, 0, 0, 0, 3858, 0] payload_rows_per_dst=[3102, 8192, 7562, 8192, 8192, 8190, 8182, 7718] dup_rows_per_dst=[56, 566, 0, 5052, 532, 0, 0, 0] combine_rows_per_src=[3102, 8192, 7562, 8192, 8192, 8190, 8182, 7718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58762 payload_rows_remote=50592 dup_rows=6774 dup_rows_remote=1838 dedup_ratio=0.103363 router_tokens_per_expert=[3262, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 11, 0, 0, 0, 4088, 0, 3530, 0, 0, 0, 0, 0, 1334, 0, 5, 0, 2, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4085, 0] payload_rows_per_dst=[6524, 7376, 8192, 8060, 4066, 8182, 8192, 8170] dup_rows_per_dst=[136, 0, 28, 1668, 4, 0, 2, 4936] combine_rows_per_src=[6524, 7376, 8192, 8060, 4066, 8182, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=181501952 dispatch_meta_bytes=206824 combine_payload_bytes=181501952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=181708776 ingress_bytes=206667776 link_bytes=206667776 token_bytes=363210728 weight_movement_bytes=25165824 total_nvlink_bytes=388376552 weight_to_token_ratio=0.069287 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=16000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.572502984e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64192 payload_rows_remote=56000 dup_rows=1344 dup_rows_remote=412 dedup_ratio=0.020508 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3585, 0, 0, 0, 0, 5, 0, 0, 0, 0, 192, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 466, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 4076, 0, 0, 1, 0, 0, 0, 0, 5, 5, 0, 1, 0, 0, 4096, 0, 3938] payload_rows_per_dst=[8192, 8176, 7180, 8192, 8192, 8180, 8192, 7888] dup_rows_per_dst=[0, 0, 0, 384, 932, 16, 12, 0] combine_rows_per_src=[8192, 8176, 7180, 8192, 8192, 8180, 8192, 7888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=228614144 dispatch_meta_bytes=229256 combine_payload_bytes=228614144 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=228843400 ingress_bytes=253779968 link_bytes=253779968 token_bytes=457457544 weight_movement_bytes=25165824 total_nvlink_bytes=482623368 weight_to_token_ratio=0.055012 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.930972330e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61542 payload_rows_remote=53350 dup_rows=3994 dup_rows_remote=3958 dedup_ratio=0.060944 router_tokens_per_expert=[2334, 1, 0, 0, 0, 0, 0, 0, 1, 16, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2685, 0, 2939, 0, 1, 0, 0, 0, 4076, 0, 14, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 4094, 175, 0, 4095, 62, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 7, 0, 0, 0, 4074, 0] payload_rows_per_dst=[8172, 8192, 5826, 6620, 8188, 8192, 8192, 8160] dup_rows_per_dst=[2376, 36, 1078, 0, 28, 352, 122, 2] combine_rows_per_src=[8172, 8192, 5826, 6620, 8188, 8192, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57382 payload_rows_remote=49190 dup_rows=8154 dup_rows_remote=8030 dedup_ratio=0.124420 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 0, 0, 0, 3, 0, 0, 0, 62, 0, 0, 4096, 0, 0, 4039, 0, 0, 0, 0, 4064, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4093] payload_rows_per_dst=[8192, 8192, 48, 8192, 8192, 8186, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 124, 8014, 16, 0, 0] combine_rows_per_src=[8192, 8192, 48, 8192, 8192, 8186, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59524 payload_rows_remote=51332 dup_rows=6012 dup_rows_remote=6012 dedup_ratio=0.091736 router_tokens_per_expert=[1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3862, 0, 4041, 0, 0, 0, 0, 0, 2748, 0, 0, 0, 16, 0, 0, 4095, 0, 0, 0, 4, 3, 4096, 223, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 0, 0, 0, 4095, 0] payload_rows_per_dst=[2750, 8180, 7736, 8090, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 20, 0, 5488, 30, 456, 0, 18] combine_rows_per_src=[2750, 8180, 7736, 8090, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=209469440 dispatch_meta_bytes=229384 combine_payload_bytes=209469440 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=209698824 ingress_bytes=284966912 link_bytes=284966912 token_bytes=419168264 weight_movement_bytes=75497472 total_nvlink_bytes=494665736 weight_to_token_ratio=0.180113 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.168268939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=207224832 dispatch_meta_bytes=209720 combine_payload_bytes=207224832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=207434552 ingress_bytes=257556480 link_bytes=257556480 token_bytes=414659384 weight_movement_bytes=50331648 total_nvlink_bytes=464991032 weight_to_token_ratio=0.121381 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.959707223e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=229376000 dispatch_meta_bytes=225648 combine_payload_bytes=229376000 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=229601648 ingress_bytes=279707648 link_bytes=279707648 token_bytes=458977648 weight_movement_bytes=50331648 total_nvlink_bytes=509309296 weight_to_token_ratio=0.109660 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23040 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.128252017e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=218521600 dispatch_meta_bytes=229232 combine_payload_bytes=218521600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=218750832 ingress_bytes=243687424 link_bytes=243687424 token_bytes=437272432 weight_movement_bytes=25165824 total_nvlink_bytes=462438256 weight_to_token_ratio=0.057552 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.854179731e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=201482240 dispatch_meta_bytes=228880 combine_payload_bytes=201482240 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=201711120 ingress_bytes=276979712 link_bytes=276979712 token_bytes=403193360 weight_movement_bytes=75497472 total_nvlink_bytes=478690832 weight_to_token_ratio=0.187249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.107495576e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=210255872 dispatch_meta_bytes=229376 combine_payload_bytes=210255872 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210485248 ingress_bytes=235421696 link_bytes=235421696 token_bytes=420741120 weight_movement_bytes=25165824 total_nvlink_bytes=445906944 weight_to_token_ratio=0.059813 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791287092e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57586 payload_rows_remote=49394 dup_rows=7950 dup_rows_remote=7946 dedup_ratio=0.121307 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1309, 0, 0, 0, 0, 0, 2463, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 3786, 179, 0, 0, 0, 0, 0, 0, 4096, 447, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 28, 0, 4096, 0, 4057] payload_rows_per_dst=[8192, 8192, 7788, 7978, 3996, 5088, 8192, 8160] dup_rows_per_dst=[4, 216, 7328, 0, 356, 0, 0, 46] combine_rows_per_src=[8192, 8192, 7788, 7978, 3996, 5088, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62152 payload_rows_remote=55552 dup_rows=3384 dup_rows_remote=2796 dedup_ratio=0.051636 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2347, 1, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 4096, 0, 0, 3878, 59, 0, 0, 0, 1238, 0, 0, 4096, 0, 0, 188, 1, 0, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 130, 30, 1, 0, 3, 0, 4095, 0, 4089] payload_rows_per_dst=[8192, 8192, 6600, 8192, 6948, 7650, 8190, 8188] dup_rows_per_dst=[0, 0, 588, 1198, 910, 370, 260, 58] combine_rows_per_src=[8192, 8192, 6600, 8192, 6948, 7650, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63246 payload_rows_remote=55102 dup_rows=2290 dup_rows_remote=2040 dedup_ratio=0.034943 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 45, 0, 0, 0, 3, 0, 4, 4096, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4096, 56, 0, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4079, 0, 4035] payload_rows_per_dst=[8158, 8036, 8188, 6406, 8192, 8186, 7936, 8144] dup_rows_per_dst=[2, 0, 86, 14, 28, 1896, 14, 250] combine_rows_per_src=[8158, 8036, 8188, 6406, 8192, 8186, 7936, 8144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=202317824 dispatch_meta_bytes=229360 combine_payload_bytes=202317824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202547184 ingress_bytes=227483648 link_bytes=227483648 token_bytes=404865008 weight_movement_bytes=25165824 total_nvlink_bytes=430030832 weight_to_token_ratio=0.062159 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.730887718e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63778 payload_rows_remote=55586 dup_rows=1758 dup_rows_remote=390 dedup_ratio=0.026825 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 2797, 30, 0, 0, 0, 0, 518, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 3988, 191, 0, 0, 0, 0, 0, 0, 4096, 110, 0, 586, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] payload_rows_per_dst=[8192, 8152, 6690, 8192, 7976, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 8, 382, 1368, 0, 0] combine_rows_per_src=[8192, 8152, 6690, 8192, 7976, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=227540992 dispatch_meta_bytes=233392 combine_payload_bytes=227540992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=227774384 ingress_bytes=252706816 link_bytes=252706816 token_bytes=455315376 weight_movement_bytes=25165824 total_nvlink_bytes=480481200 weight_to_token_ratio=0.055271 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=20480 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.922806883e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65484 payload_rows_remote=57292 dup_rows=52 dup_rows_remote=48 dedup_ratio=0.000793 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 0, 1, 0, 4078, 0, 0, 4094, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4090, 0, 0] payload_rows_per_dst=[8192, 8158, 8192, 8190, 8192, 8188, 8192, 8180] dup_rows_per_dst=[8, 0, 22, 0, 4, 14, 4, 0] combine_rows_per_src=[8192, 8158, 8192, 8190, 8192, 8188, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57864 payload_rows_remote=49676 dup_rows=7672 dup_rows_remote=7672 dedup_ratio=0.117065 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 10, 7, 0, 0, 0, 0, 4093, 1, 0, 0, 3427, 0, 0, 0, 0, 0, 340, 5, 2, 0, 137, 0, 0, 4092, 1, 0, 3953, 305, 0, 1, 1, 0, 1, 0, 4094, 0, 0, 12, 0, 0, 0, 4066, 20, 0, 0, 0, 0, 0, 0, 10, 8, 0, 0, 0, 2, 4092, 0, 4085] payload_rows_per_dst=[8192, 8188, 8178, 8188, 614, 8132, 8190, 8182] dup_rows_per_dst=[36, 0, 7272, 276, 2, 24, 54, 8] combine_rows_per_src=[8192, 8188, 8178, 8188, 614, 8132, 8190, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65534 payload_rows_remote=57342 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62836 payload_rows_remote=54724 dup_rows=2700 dup_rows_remote=2700 dedup_ratio=0.041199 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 2791, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1305, 0, 0, 4089, 0, 0, 19, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4056, 0, 4095] payload_rows_per_dst=[8184, 8192, 5582, 8192, 8192, 8190, 8112, 8192] dup_rows_per_dst=[0, 0, 8, 0, 2610, 42, 0, 40] combine_rows_per_src=[8184, 8192, 5582, 8192, 8192, 8190, 8112, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=225697792 dispatch_meta_bytes=228568 combine_payload_bytes=225697792 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=225926360 ingress_bytes=276029440 link_bytes=276029440 token_bytes=451624152 weight_movement_bytes=50331648 total_nvlink_bytes=501955800 weight_to_token_ratio=0.111446 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.100265104e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=227680256 dispatch_meta_bytes=223904 combine_payload_bytes=227680256 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=227904160 ingress_bytes=303177728 link_bytes=303177728 token_bytes=455584416 weight_movement_bytes=75497472 total_nvlink_bytes=531081888 weight_to_token_ratio=0.165716 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.306832208e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234668032 dispatch_meta_bytes=229360 combine_payload_bytes=234668032 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234897392 ingress_bytes=259833856 link_bytes=259833856 token_bytes=469565424 weight_movement_bytes=25165824 total_nvlink_bytes=494731248 weight_to_token_ratio=0.053594 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.977035423e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=203472896 dispatch_meta_bytes=229392 combine_payload_bytes=203472896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203702288 ingress_bytes=228638720 link_bytes=228638720 token_bytes=407175184 weight_movement_bytes=25165824 total_nvlink_bytes=432341008 weight_to_token_ratio=0.061806 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.739676482e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234872832 dispatch_meta_bytes=229376 combine_payload_bytes=234872832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=235102208 ingress_bytes=285204480 link_bytes=285204480 token_bytes=469975040 weight_movement_bytes=50331648 total_nvlink_bytes=520306688 weight_to_token_ratio=0.107094 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.170076557e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=224149504 dispatch_meta_bytes=229696 combine_payload_bytes=224149504 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=224379200 ingress_bytes=249315328 link_bytes=249315328 token_bytes=448528704 weight_movement_bytes=25165824 total_nvlink_bytes=473694528 weight_to_token_ratio=0.056107 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.897001578e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63544 payload_rows_remote=55352 dup_rows=1992 dup_rows_remote=1992 dedup_ratio=0.030396 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 63, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 2681, 0, 89, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7514, 7298, 7912, 8052, 8192] dup_rows_per_dst=[0, 1072, 266, 6, 0, 644, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 7514, 7298, 7912, 8052, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65514 payload_rows_remote=57322 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4, 0, 0, 334, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 8192, 8192, 8192, 8192, 8182] dup_rows_per_dst=[0, 0, 0, 0, 10, 12, 0, 0] combine_rows_per_src=[8192, 8180, 8192, 8192, 8192, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=226721792 dispatch_meta_bytes=229376 combine_payload_bytes=226721792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=226951168 ingress_bytes=251887616 link_bytes=251887616 token_bytes=453672960 weight_movement_bytes=25165824 total_nvlink_bytes=478838784 weight_to_token_ratio=0.055471 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.916573718e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64150 payload_rows_remote=55958 dup_rows=1386 dup_rows_remote=1084 dedup_ratio=0.021149 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 4089, 0, 0, 4096, 4, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4044, 3, 0, 0, 0, 9, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8184, 7226, 7884, 8088, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 22, 938, 24, 98, 0, 302] combine_rows_per_src=[8184, 7226, 7884, 8088, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=229376 combine_payload_bytes=234790912 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=235020288 ingress_bytes=310288384 link_bytes=310288384 token_bytes=469811200 weight_movement_bytes=75497472 total_nvlink_bytes=545308672 weight_to_token_ratio=0.160697 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.360936084e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65164 payload_rows_remote=57064 dup_rows=372 dup_rows_remote=372 dedup_ratio=0.005676 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 4094, 4094, 0, 0, 0, 4050, 0, 0, 4094, 0, 0, 4091, 0, 0, 0, 13, 0, 0, 3950, 1, 0, 0, 1, 4067, 0, 27, 0, 0, 0, 136, 0, 1, 59, 1, 1, 2, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 2, 0, 0, 4072, 2, 0] payload_rows_per_dst=[8190, 8100, 8190, 8136, 8192, 8020, 8188, 8148] dup_rows_per_dst=[0, 0, 18, 2, 324, 16, 4, 8] combine_rows_per_src=[8190, 8100, 8190, 8136, 8192, 8020, 8188, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65186 payload_rows_remote=57118 dup_rows=350 dup_rows_remote=304 dedup_ratio=0.005341 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 18, 0, 4053, 0, 0, 4096, 0, 0, 4067, 0, 0, 0, 0, 1, 0, 4033, 0, 0, 0, 0, 4088, 0, 56, 0, 0, 0, 59, 0, 0, 2, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4045, 0, 0] payload_rows_per_dst=[8192, 8142, 8134, 8176, 8192, 8068, 8192, 8090] dup_rows_per_dst=[2, 0, 0, 2, 230, 46, 70, 0] combine_rows_per_src=[8192, 8142, 8134, 8176, 8192, 8068, 8192, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63902 payload_rows_remote=55732 dup_rows=1634 dup_rows_remote=550 dedup_ratio=0.024933 router_tokens_per_expert=[7, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4085, 4075, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 544, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 4096, 15, 4096, 0, 0, 4, 0] payload_rows_per_dst=[8192, 7700, 8192, 8192, 8170, 8192, 7072, 8192] dup_rows_per_dst=[16, 0, 0, 0, 1084, 28, 468, 38] combine_rows_per_src=[8192, 7700, 8192, 8192, 8170, 8192, 7072, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57558 payload_rows_remote=49366 dup_rows=7978 dup_rows_remote=7978 dedup_ratio=0.121735 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 3, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 214, 8192] dup_rows_per_dst=[0, 6, 48, 0, 7924, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 214, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65052 payload_rows_remote=56860 dup_rows=484 dup_rows_remote=422 dedup_ratio=0.007385 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 78, 0, 4058, 0, 0, 4096, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 3967, 0, 0, 0, 0, 4050, 0, 26, 0, 0, 0, 125, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8100, 8192, 7940, 8192, 8052] dup_rows_per_dst=[0, 80, 40, 0, 302, 0, 62, 0] combine_rows_per_src=[8192, 8192, 8192, 8100, 8192, 7940, 8192, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=229203968 dispatch_meta_bytes=228168 combine_payload_bytes=229203968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229432136 ingress_bytes=254369792 link_bytes=254369792 token_bytes=458636104 weight_movement_bytes=25165824 total_nvlink_bytes=483801928 weight_to_token_ratio=0.054871 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.935460209e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233734144 dispatch_meta_bytes=229744 combine_payload_bytes=233734144 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=233963888 ingress_bytes=334397440 link_bytes=334397440 token_bytes=467698032 weight_movement_bytes=100663296 total_nvlink_bytes=568361328 weight_to_token_ratio=0.215231 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.544378144e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233955328 dispatch_meta_bytes=229688 combine_payload_bytes=233955328 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234185016 ingress_bytes=259121152 link_bytes=259121152 token_bytes=468140344 weight_movement_bytes=25165824 total_nvlink_bytes=493306168 weight_to_token_ratio=0.053757 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.971612569e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228278272 dispatch_meta_bytes=225128 combine_payload_bytes=228278272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=228503400 ingress_bytes=253444096 link_bytes=253444096 token_bytes=456781672 weight_movement_bytes=25165824 total_nvlink_bytes=481947496 weight_to_token_ratio=0.055094 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.928416732e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=202203136 dispatch_meta_bytes=229376 combine_payload_bytes=202203136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202432512 ingress_bytes=227368960 link_bytes=227368960 token_bytes=404635648 weight_movement_bytes=25165824 total_nvlink_bytes=429801472 weight_to_token_ratio=0.062194 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.730015075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232898560 dispatch_meta_bytes=229128 combine_payload_bytes=232898560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233127688 ingress_bytes=258064384 link_bytes=258064384 token_bytes=466026248 weight_movement_bytes=25165824 total_nvlink_bytes=491192072 weight_to_token_ratio=0.054001 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.963571785e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50000 payload_rows_remote=41836 dup_rows=15536 dup_rows_remote=15496 dedup_ratio=0.237061 router_tokens_per_expert=[0, 4082, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3943, 4089, 0, 5, 0, 0, 0, 4040, 0, 463, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 175, 0, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 0, 18, 0, 0, 2420, 0, 0, 4073, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8164, 8192, 8192, 8192, 350, 8192, 4876, 3842] dup_rows_per_dst=[40, 12208, 774, 0, 4, 2510, 0, 0] combine_rows_per_src=[8164, 8192, 8192, 8192, 350, 8192, 4876, 3842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60740 payload_rows_remote=52772 dup_rows=4796 dup_rows_remote=4796 dedup_ratio=0.073181 router_tokens_per_expert=[2, 4086, 0, 0, 0, 1, 0, 0, 117, 0, 0, 4090, 4095, 0, 7, 0, 0, 0, 2, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 42, 0, 0, 3886, 0, 0, 4087, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 7968, 8188, 3994, 8192, 7856, 8174] dup_rows_per_dst=[2, 4760, 0, 0, 24, 10, 0, 0] combine_rows_per_src=[8176, 8192, 7968, 8188, 3994, 8192, 7856, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=7016401512 prefetch_bytes=427819008 reduce_bytes=276824064 weight_movement_bytes=704643072 egress_bytes=3510028904 ingress_bytes=4211015680 link_bytes=4211015680 total_nvlink_bytes=7721044584 weight_to_token_ratio=0.100428 prefetch_share_of_ingress=0.101595 dedup_ratio=0.067785 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=11 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=16637 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.002560235e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6723773256 prefetch_bytes=327155712 reduce_bytes=452984832 weight_movement_bytes=780140544 egress_bytes=3363750728 ingress_bytes=4140163072 link_bytes=4140163072 total_nvlink_bytes=7503913800 weight_to_token_ratio=0.116027 prefetch_share_of_ingress=0.079020 dedup_ratio=0.110563 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=18 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.043428 gemm_inner_pad_upper_bound=15240 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.968866080e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=171360256 dispatch_meta_bytes=229328 combine_payload_bytes=171360256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=171589584 ingress_bytes=196526080 link_bytes=196526080 token_bytes=342949840 weight_movement_bytes=25165824 total_nvlink_bytes=368115664 weight_to_token_ratio=0.073380 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.495336395e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52860 payload_rows_remote=44732 dup_rows=12676 dup_rows_remote=8306 dedup_ratio=0.193420 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 176, 0, 3999, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 3162, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 5, 27, 0, 0, 797, 0, 0, 4064, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8188, 2, 7842, 8192, 8192, 4124, 8192, 8128] dup_rows_per_dst=[0, 0, 164, 46, 6330, 108, 1658, 4370] combine_rows_per_src=[8188, 2, 7842, 8192, 8192, 4124, 8192, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=216154112 dispatch_meta_bytes=230272 combine_payload_bytes=216154112 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=216384384 ingress_bytes=291651584 link_bytes=291651584 token_bytes=432538496 weight_movement_bytes=75497472 total_nvlink_bytes=508035968 weight_to_token_ratio=0.174545 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.219131569e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57174 payload_rows_remote=48990 dup_rows=8362 dup_rows_remote=264 dedup_ratio=0.127594 router_tokens_per_expert=[2, 4093, 10, 0, 0, 0, 1, 3, 0, 0, 0, 4049, 4092, 0, 0, 0, 0, 0, 0, 16, 4087, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 17, 0, 2, 0, 0, 0, 6, 1546, 0, 0, 34, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 0, 4079, 3, 4093, 0, 78, 3, 0] payload_rows_per_dst=[8192, 8184, 8180, 8152, 3134, 8192, 4948, 8192] dup_rows_per_dst=[26, 8098, 26, 0, 8, 70, 0, 134] combine_rows_per_src=[8192, 8184, 8180, 8152, 3134, 8192, 4948, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56804 payload_rows_remote=48628 dup_rows=8732 dup_rows_remote=7444 dedup_ratio=0.133240 router_tokens_per_expert=[0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 3, 3880, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 562, 0, 0, 536, 111, 0, 1, 0, 0, 0, 0, 0, 211, 0, 7, 2912, 0, 0, 4062, 0, 4084, 0, 23, 0, 0] payload_rows_per_dst=[8156, 0, 7766, 8192, 8172, 8176, 8192, 8150] dup_rows_per_dst=[0, 0, 0, 28, 1136, 1288, 6260, 20] combine_rows_per_src=[8156, 0, 7766, 8192, 8172, 8176, 8192, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5810175672 prefetch_bytes=327155712 reduce_bytes=452984832 weight_movement_bytes=780140544 egress_bytes=2906799800 ingress_bytes=3683516416 link_bytes=3683516416 total_nvlink_bytes=6590316216 weight_to_token_ratio=0.134271 prefetch_share_of_ingress=0.088816 dedup_ratio=0.206177 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=18 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=15621 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.751706491e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57228 payload_rows_remote=49036 dup_rows=8308 dup_rows_remote=700 dedup_ratio=0.126770 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 314, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 4050, 0, 4096, 0, 0, 1, 1] payload_rows_per_dst=[8192, 2, 8192, 8162, 8192, 8192, 8192, 8104] dup_rows_per_dst=[0, 0, 0, 0, 630, 70, 7608, 0] combine_rows_per_src=[8192, 2, 8192, 8162, 8192, 8192, 8192, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=183222272 dispatch_meta_bytes=212152 combine_payload_bytes=183222272 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=183434424 ingress_bytes=258719744 link_bytes=258719744 token_bytes=366656696 weight_movement_bytes=75497472 total_nvlink_bytes=442154168 weight_to_token_ratio=0.205908 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.968558318e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6699612816 prefetch_bytes=276824064 reduce_bytes=503316480 weight_movement_bytes=780140544 egress_bytes=3351624336 ingress_bytes=4128129024 link_bytes=4128129024 total_nvlink_bytes=7479753360 weight_to_token_ratio=0.116446 prefetch_share_of_ingress=0.067058 dedup_ratio=0.099260 prefetch_slots=22 unused_prefetch_slots=234 slot_utilization=0.085938 reduce_slots=20 remote_demand=11 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=15367 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.963143255e-04 prefetch_bytes_per_gemm_flop=1.316444547e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=200663040 dispatch_meta_bytes=197016 combine_payload_bytes=200663040 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=200860056 ingress_bytes=276160512 link_bytes=276160512 token_bytes=401523096 weight_movement_bytes=75497472 total_nvlink_bytes=477020568 weight_to_token_ratio=0.188028 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=20096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.101262410e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=199180288 dispatch_meta_bytes=224288 combine_payload_bytes=199180288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199404576 ingress_bytes=224346112 link_bytes=224346112 token_bytes=398584864 weight_movement_bytes=25165824 total_nvlink_bytes=423750688 weight_to_token_ratio=0.063138 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.707014695e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=200851456 dispatch_meta_bytes=198944 combine_payload_bytes=200851456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201050400 ingress_bytes=226017280 link_bytes=226017280 token_bytes=401901856 weight_movement_bytes=25165824 total_nvlink_bytes=427067680 weight_to_token_ratio=0.062617 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.719730352e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5977900184 prefetch_bytes=377487360 reduce_bytes=276824064 weight_movement_bytes=654311424 egress_bytes=2990679192 ingress_bytes=3641532416 link_bytes=3641532416 total_nvlink_bytes=6632211608 weight_to_token_ratio=0.109455 prefetch_share_of_ingress=0.103662 dedup_ratio=0.183300 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=11 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=16129 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.731740883e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6394705824 prefetch_bytes=377487360 reduce_bytes=327155712 weight_movement_bytes=704643072 egress_bytes=3199129504 ingress_bytes=3900219392 link_bytes=3900219392 total_nvlink_bytes=7099348896 weight_to_token_ratio=0.110192 prefetch_share_of_ingress=0.096786 dedup_ratio=0.140894 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=13 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=15113 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.854760195e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6600564296 prefetch_bytes=377487360 reduce_bytes=478150656 weight_movement_bytes=855638016 egress_bytes=3302104648 ingress_bytes=4154097664 link_bytes=4154097664 total_nvlink_bytes=7456202312 weight_to_token_ratio=0.129631 prefetch_share_of_ingress=0.090871 dedup_ratio=0.118376 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=19 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=16510 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.975492714e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6562488264 prefetch_bytes=427819008 reduce_bytes=150994944 weight_movement_bytes=578813952 egress_bytes=3283025864 ingress_bytes=3858276352 link_bytes=3858276352 total_nvlink_bytes=7141302216 weight_to_token_ratio=0.088200 prefetch_share_of_ingress=0.110883 dedup_ratio=0.120663 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=6 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=16510 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.834814065e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none + [2026-09-21 08:31:45] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 220.0 | throughput per GPU (TFLOP/s/GPU): 146.5 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.625602E+00 | loss scale: 1.0 | grad norm: 8.176 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45342 payload_rows_remote=39268 dup_rows=20194 dup_rows_remote=17128 dedup_ratio=0.308136 router_tokens_per_expert=[197, 522, 444, 4, 1, 703, 1262, 68, 1, 1666, 38, 1, 6, 101, 311, 263, 10, 734, 408, 1831, 19, 180, 312, 28, 197, 27, 48, 588, 998, 444, 912, 1793, 323, 335, 594, 583, 255, 1369, 470, 1002, 867, 803, 1769, 17, 1603, 885, 18, 61, 659, 673, 611, 43, 386, 966, 1777, 45, 1765, 15, 214, 2, 8, 30, 473, 30] payload_rows_per_dst=[6074, 6778, 6830, 4882, 5046, 4420, 4914, 6398] dup_rows_per_dst=[3066, 4740, 3800, 1546, 2078, 882, 1852, 2230] combine_rows_per_src=[6074, 6778, 6830, 4882, 5046, 4420, 4914, 6398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46980 payload_rows_remote=40382 dup_rows=18556 dup_rows_remote=14912 dedup_ratio=0.283142 router_tokens_per_expert=[284, 519, 362, 1, 0, 585, 1625, 119, 2, 1365, 28, 0, 2, 26, 176, 260, 12, 897, 416, 2043, 19, 217, 268, 14, 257, 41, 79, 251, 1419, 884, 653, 1537, 395, 186, 443, 778, 150, 1761, 248, 639, 1005, 1056, 1619, 45, 1133, 803, 9, 265, 786, 808, 534, 39, 316, 1272, 1686, 22, 1663, 10, 240, 14, 8, 38, 413, 23] payload_rows_per_dst=[5694, 5446, 5694, 6598, 5792, 6202, 5314, 6240] dup_rows_per_dst=[2348, 1510, 2078, 3644, 2356, 2430, 2240, 1950] combine_rows_per_src=[5694, 5446, 5694, 6598, 5792, 6202, 5314, 6240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46332 payload_rows_remote=40476 dup_rows=19204 dup_rows_remote=17198 dedup_ratio=0.293030 router_tokens_per_expert=[257, 523, 508, 1, 2, 837, 1024, 244, 9, 1479, 19, 1, 28, 159, 467, 316, 4, 997, 401, 1994, 15, 293, 210, 17, 209, 21, 65, 303, 909, 930, 726, 1487, 344, 368, 553, 610, 99, 2012, 254, 1066, 712, 960, 1390, 32, 1678, 759, 18, 161, 769, 500, 693, 32, 393, 1200, 1409, 71, 1398, 16, 251, 4, 11, 52, 465, 33] payload_rows_per_dst=[6650, 6052, 5856, 6330, 4806, 5930, 5126, 5582] dup_rows_per_dst=[4166, 2396, 2006, 2970, 1782, 1998, 2190, 1696] combine_rows_per_src=[6650, 6052, 5856, 6330, 4806, 5930, 5126, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46964 payload_rows_remote=40288 dup_rows=18572 dup_rows_remote=15190 dedup_ratio=0.283386 router_tokens_per_expert=[408, 303, 491, 4, 0, 728, 1063, 169, 1, 1466, 9, 6, 3, 108, 63, 153, 1, 946, 570, 1627, 28, 501, 256, 11, 121, 6, 25, 402, 936, 753, 726, 1649, 211, 183, 412, 856, 74, 1945, 236, 1112, 1058, 738, 1612, 21, 1581, 1111, 12, 158, 929, 529, 826, 15, 312, 1030, 1779, 57, 1649, 19, 341, 3, 14, 9, 379, 24] payload_rows_per_dst=[4790, 5404, 5840, 6262, 6676, 6490, 5250, 6252] dup_rows_per_dst=[1542, 1438, 2040, 2974, 3382, 2868, 2146, 2182] combine_rows_per_src=[4790, 5404, 5840, 6262, 6676, 6490, 5250, 6252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46632 payload_rows_remote=40248 dup_rows=18904 dup_rows_remote=16180 dedup_ratio=0.288452 router_tokens_per_expert=[285, 429, 356, 3, 2, 526, 1519, 217, 2, 1302, 31, 0, 59, 40, 128, 151, 48, 1023, 772, 2155, 27, 402, 242, 29, 241, 31, 10, 260, 1135, 1419, 526, 1502, 357, 253, 475, 483, 177, 1277, 233, 814, 995, 1294, 1831, 45, 1388, 540, 28, 264, 705, 600, 776, 48, 212, 1480, 1383, 58, 1099, 16, 440, 12, 23, 128, 435, 27] payload_rows_per_dst=[5030, 5636, 6498, 6428, 5764, 6384, 5496, 5396] dup_rows_per_dst=[1644, 1452, 2898, 3684, 2374, 2724, 2398, 1730] combine_rows_per_src=[5030, 5636, 6498, 6428, 5764, 6384, 5496, 5396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46508 payload_rows_remote=39832 dup_rows=19028 dup_rows_remote=14458 dedup_ratio=0.290344 router_tokens_per_expert=[355, 395, 345, 3, 1, 499, 1303, 167, 3, 1051, 36, 2, 12, 39, 193, 162, 4, 1258, 731, 1346, 26, 570, 213, 5, 162, 5, 73, 253, 1198, 813, 660, 1143, 245, 198, 547, 653, 89, 1917, 290, 897, 1054, 972, 1595, 27, 1470, 1174, 24, 192, 1005, 662, 722, 58, 373, 1555, 1685, 105, 1334, 19, 251, 2, 17, 98, 482, 30] payload_rows_per_dst=[4724, 5080, 5996, 4944, 6484, 6700, 6676, 5904] dup_rows_per_dst=[1412, 1106, 2310, 1384, 3188, 3126, 4570, 1932] combine_rows_per_src=[4724, 5080, 5996, 4944, 6484, 6700, 6676, 5904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45930 payload_rows_remote=39300 dup_rows=19606 dup_rows_remote=16054 dedup_ratio=0.299164 router_tokens_per_expert=[353, 618, 306, 2, 0, 463, 1633, 75, 9, 1055, 68, 4, 16, 13, 199, 319, 14, 914, 551, 1424, 35, 68, 321, 19, 288, 12, 22, 381, 1385, 413, 778, 1560, 312, 224, 574, 430, 195, 1640, 385, 604, 1025, 910, 2085, 11, 1192, 1090, 8, 113, 665, 1198, 662, 65, 563, 896, 1554, 15, 1591, 10, 358, 32, 9, 133, 889, 12] payload_rows_per_dst=[5314, 5762, 5152, 3832, 6420, 6460, 6360, 6630] dup_rows_per_dst=[1586, 1774, 1540, 824, 3206, 3242, 3882, 3552] combine_rows_per_src=[5314, 5762, 5152, 3832, 6420, 6460, 6360, 6630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160841728 dispatch_meta_bytes=225584 combine_payload_bytes=160841728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=161067312 ingress_bytes=186007552 link_bytes=186007552 token_bytes=321909040 weight_movement_bytes=25165824 total_nvlink_bytes=347074864 weight_to_token_ratio=0.078177 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.415302551e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165404672 dispatch_meta_bytes=221176 combine_payload_bytes=165404672 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=165625848 ingress_bytes=266067968 link_bytes=266067968 token_bytes=331030520 weight_movement_bytes=100663296 total_nvlink_bytes=431693816 weight_to_token_ratio=0.304091 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.024469812e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165789696 dispatch_meta_bytes=230696 combine_payload_bytes=165789696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166020392 ingress_bytes=190955520 link_bytes=190955520 token_bytes=331810088 weight_movement_bytes=25165824 total_nvlink_bytes=356975912 weight_to_token_ratio=0.075844 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.452950870e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165019648 dispatch_meta_bytes=221912 combine_payload_bytes=165019648 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=165241560 ingress_bytes=215351296 link_bytes=215351296 token_bytes=330261208 weight_movement_bytes=50331648 total_nvlink_bytes=380592856 weight_to_token_ratio=0.152400 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.638574538e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45578 payload_rows_remote=38902 dup_rows=19958 dup_rows_remote=15672 dedup_ratio=0.304535 router_tokens_per_expert=[305, 306, 599, 19, 0, 543, 1462, 181, 14, 1305, 42, 0, 22, 16, 302, 261, 7, 808, 619, 1518, 23, 84, 192, 1, 86, 12, 65, 317, 1215, 975, 555, 1639, 604, 297, 541, 633, 88, 2333, 184, 744, 1038, 821, 2156, 29, 1363, 780, 25, 147, 941, 530, 738, 46, 198, 1030, 1426, 78, 1739, 39, 356, 15, 11, 23, 307, 15] payload_rows_per_dst=[7134, 6676, 5258, 6232, 4604, 4606, 5092, 5976] dup_rows_per_dst=[4362, 4286, 1740, 3002, 1578, 1074, 2030, 1886] combine_rows_per_src=[7134, 6676, 5258, 6232, 4604, 4606, 5092, 5976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164855808 dispatch_meta_bytes=225712 combine_payload_bytes=164855808 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=165081520 ingress_bytes=240353280 link_bytes=240353280 token_bytes=329937328 weight_movement_bytes=75497472 total_nvlink_bytes=405434800 weight_to_token_ratio=0.228824 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.828810748e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163151872 dispatch_meta_bytes=217160 combine_payload_bytes=163151872 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=163369032 ingress_bytes=238649344 link_bytes=238649344 token_bytes=326520904 weight_movement_bytes=75497472 total_nvlink_bytes=402018376 weight_to_token_ratio=0.231218 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.815845764e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160972800 dispatch_meta_bytes=221416 combine_payload_bytes=160972800 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=161194216 ingress_bytes=211304448 link_bytes=211304448 token_bytes=322167016 weight_movement_bytes=50331648 total_nvlink_bytes=372498664 weight_to_token_ratio=0.156228 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=12800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.607782700e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=218296 combine_payload_bytes=159342592 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=159560888 ingress_bytes=209674240 link_bytes=209674240 token_bytes=318903480 weight_movement_bytes=50331648 total_nvlink_bytes=369235128 weight_to_token_ratio=0.157827 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=14080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595378701e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54006 payload_rows_remote=46042 dup_rows=11530 dup_rows_remote=11490 dedup_ratio=0.175934 router_tokens_per_expert=[2, 4, 17, 1, 6, 5, 0, 0, 100, 0, 2696, 95, 0, 0, 1840, 0, 1, 0, 3942, 3, 86, 4, 2644, 9, 0, 656, 0, 0, 197, 0, 2632, 742, 7, 1, 1, 0, 56, 0, 3618, 0, 3967, 0, 3244, 1, 2, 0, 0, 0, 303, 4, 4, 20, 3, 2, 1, 0, 2493, 0, 577, 559, 1796, 3, 424, 0] payload_rows_per_dst=[7964, 6924, 8032, 6956, 7346, 6494, 5734, 4556] dup_rows_per_dst=[40, 2538, 5346, 1498, 20, 0, 1044, 1044] combine_rows_per_src=[7964, 6924, 8032, 6956, 7346, 6494, 5734, 4556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52052 payload_rows_remote=45552 dup_rows=13484 dup_rows_remote=12208 dedup_ratio=0.205750 router_tokens_per_expert=[92, 0, 444, 1, 0, 0, 0, 0, 0, 0, 1210, 123, 1, 0, 1360, 6, 0, 0, 4094, 0, 1, 16, 2879, 0, 0, 457, 0, 0, 35, 0, 2659, 737, 1, 1, 0, 0, 0, 0, 3162, 1, 4096, 0, 4087, 0, 0, 0, 7, 0, 1162, 0, 4, 0, 2, 1, 0, 7, 2175, 0, 1554, 2212, 161, 0, 20, 0] payload_rows_per_dst=[8192, 4462, 6434, 6500, 7228, 8178, 6502, 4556] dup_rows_per_dst=[1074, 938, 2762, 1276, 3886, 10, 2782, 756] combine_rows_per_src=[8192, 4462, 6434, 6500, 7228, 8178, 6502, 4556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53956 payload_rows_remote=45856 dup_rows=11580 dup_rows_remote=4164 dedup_ratio=0.176697 router_tokens_per_expert=[0, 0, 280, 0, 0, 4, 0, 1, 4, 53, 158, 80, 0, 0, 3557, 0, 165, 0, 4027, 3, 0, 111, 1601, 16, 1, 44, 0, 0, 14, 0, 3600, 526, 0, 0, 0, 0, 6, 0, 3724, 1, 4088, 3, 3792, 1, 5, 0, 0, 6, 711, 466, 0, 0, 0, 0, 5, 0, 3124, 15, 1280, 1287, 7, 0, 2, 0] payload_rows_per_dst=[8184, 7250, 3566, 7442, 8100, 7600, 7130, 4684] dup_rows_per_dst=[562, 454, 226, 928, 7416, 14, 1482, 498] combine_rows_per_src=[8184, 7250, 3566, 7442, 8100, 7600, 7130, 4684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52508 payload_rows_remote=44700 dup_rows=13028 dup_rows_remote=11920 dedup_ratio=0.198792 router_tokens_per_expert=[113, 0, 446, 2, 17, 0, 0, 0, 0, 0, 124, 1097, 0, 0, 3476, 0, 1, 4, 3898, 0, 4, 9, 542, 0, 0, 307, 0, 1, 0, 0, 3390, 686, 0, 0, 15, 0, 6, 138, 1541, 4, 3872, 0, 4024, 0, 7, 0, 88, 0, 1009, 0, 0, 0, 78, 0, 1, 0, 3436, 2, 934, 3209, 271, 3, 13, 0] payload_rows_per_dst=[8088, 7534, 7808, 7426, 3140, 8070, 8092, 2350] dup_rows_per_dst=[812, 1860, 1108, 1342, 268, 168, 7374, 96] combine_rows_per_src=[8088, 7534, 7808, 7426, 3140, 8070, 8092, 2350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48978 payload_rows_remote=43508 dup_rows=16558 dup_rows_remote=16426 dedup_ratio=0.252655 router_tokens_per_expert=[50, 1, 202, 0, 0, 62, 0, 0, 88, 0, 1515, 438, 45, 0, 64, 0, 17, 0, 4054, 0, 896, 19, 3655, 1, 0, 203, 0, 11, 397, 19, 2409, 184, 4, 0, 0, 0, 10, 0, 4048, 27, 4095, 8, 2691, 0, 0, 0, 100, 2, 101, 0, 15, 3, 0, 0, 1, 0, 947, 1, 3459, 1359, 73, 0, 1494, 0] payload_rows_per_dst=[8190, 4068, 7510, 5740, 8172, 5470, 2036, 7792] dup_rows_per_dst=[630, 232, 1666, 706, 7858, 132, 98, 5236] combine_rows_per_src=[8190, 4068, 7510, 5740, 8172, 5470, 2036, 7792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56002 payload_rows_remote=47946 dup_rows=9534 dup_rows_remote=8420 dedup_ratio=0.145477 router_tokens_per_expert=[1, 8, 117, 0, 0, 2, 0, 0, 9, 0, 1129, 949, 86, 0, 316, 0, 0, 0, 4079, 0, 40, 52, 2248, 14, 0, 0, 0, 3, 2, 0, 4066, 186, 4, 0, 0, 0, 0, 20, 3106, 2, 4087, 0, 4031, 1, 4, 0, 0, 0, 431, 5, 196, 3, 0, 0, 0, 0, 3950, 2, 1527, 1794, 257, 0, 40, 1] payload_rows_per_dst=[8190, 4534, 4620, 8140, 6226, 8064, 8056, 8172] dup_rows_per_dst=[240, 444, 88, 374, 38, 8, 1114, 7228] combine_rows_per_src=[8190, 4534, 4620, 8140, 6226, 8064, 8056, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46342 payload_rows_remote=38156 dup_rows=19194 dup_rows_remote=11910 dedup_ratio=0.292877 router_tokens_per_expert=[11, 1, 605, 0, 0, 2, 0, 2, 3, 4, 3266, 1549, 7, 0, 1369, 14, 9, 0, 4091, 5, 0, 0, 1468, 21, 0, 84, 0, 4, 16, 128, 3303, 472, 1, 13, 0, 0, 0, 0, 560, 55, 3972, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 56, 3941, 1, 1053, 1180, 1130, 0, 280, 0] payload_rows_per_dst=[4306, 7604, 1704, 7240, 1236, 8172, 7894, 8186] dup_rows_per_dst=[986, 3656, 136, 1710, 22, 5288, 112, 7284] combine_rows_per_src=[4306, 7604, 1704, 7240, 1236, 8172, 7894, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=188588032 dispatch_meta_bytes=230128 combine_payload_bytes=188588032 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188818160 ingress_bytes=213753856 link_bytes=213753856 token_bytes=377406192 weight_movement_bytes=25165824 total_nvlink_bytes=402572016 weight_to_token_ratio=0.066681 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.626419865e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=186580992 dispatch_meta_bytes=231040 combine_payload_bytes=186580992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186812032 ingress_bytes=211746816 link_bytes=211746816 token_bytes=373393024 weight_movement_bytes=25165824 total_nvlink_bytes=398558848 weight_to_token_ratio=0.067398 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.611148610e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=187826176 dispatch_meta_bytes=200080 combine_payload_bytes=187826176 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188026256 ingress_bytes=212992000 link_bytes=212992000 token_bytes=375852432 weight_movement_bytes=25165824 total_nvlink_bytes=401018256 weight_to_token_ratio=0.066957 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.620623021e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=183091200 dispatch_meta_bytes=226480 combine_payload_bytes=183091200 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=183317680 ingress_bytes=283754496 link_bytes=283754496 token_bytes=366408880 weight_movement_bytes=100663296 total_nvlink_bytes=467072176 weight_to_token_ratio=0.274729 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.159043854e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=178208768 dispatch_meta_bytes=239736 combine_payload_bytes=178208768 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=178448504 ingress_bytes=228540416 link_bytes=228540416 token_bytes=356657272 weight_movement_bytes=50331648 total_nvlink_bytes=406988920 weight_to_token_ratio=0.141120 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.738928502e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=196386816 dispatch_meta_bytes=225464 combine_payload_bytes=196386816 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=196612280 ingress_bytes=246718464 link_bytes=246718464 token_bytes=392999096 weight_movement_bytes=50331648 total_nvlink_bytes=443330744 weight_to_token_ratio=0.128071 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=20864 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.877242444e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52450 payload_rows_remote=44434 dup_rows=13086 dup_rows_remote=8064 dedup_ratio=0.199677 router_tokens_per_expert=[1, 1, 81, 0, 4, 30, 0, 17, 0, 236, 2867, 253, 3, 0, 3160, 0, 0, 0, 3974, 0, 0, 10, 3221, 60, 0, 12, 0, 0, 6, 0, 3981, 130, 0, 0, 12, 0, 79, 0, 2199, 0, 3905, 0, 3930, 0, 1, 0, 1, 2, 4, 0, 3, 8, 0, 2, 1, 0, 752, 17, 757, 2273, 702, 0, 73, 0] payload_rows_per_dst=[7854, 8016, 8150, 7966, 4570, 7864, 5484, 2546] dup_rows_per_dst=[224, 5022, 6380, 292, 10, 4, 602, 552] combine_rows_per_src=[7854, 8016, 8150, 7966, 4570, 7864, 5484, 2546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=156286976 dispatch_meta_bytes=200264 combine_payload_bytes=156286976 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=156487240 ingress_bytes=231784448 link_bytes=231784448 token_bytes=312774216 weight_movement_bytes=75497472 total_nvlink_bytes=388271688 weight_to_token_ratio=0.241380 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.763611837e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=182001664 dispatch_meta_bytes=209992 combine_payload_bytes=182001664 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=182211656 ingress_bytes=232333312 link_bytes=232333312 token_bytes=364213320 weight_movement_bytes=50331648 total_nvlink_bytes=414544968 weight_to_token_ratio=0.138193 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=14592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.767788058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=34078 payload_rows_remote=25902 dup_rows=31458 dup_rows_remote=17752 dedup_ratio=0.480011 router_tokens_per_expert=[1, 11, 0, 0, 3875, 0, 4075, 0, 0, 4, 0, 3943, 0, 0, 2, 3930, 3949, 0, 0, 3947, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1466, 0, 0, 0, 2, 7, 0, 1, 0, 0, 0, 0, 0, 5, 0, 2, 89, 2914, 0, 0, 4043, 4, 180, 5, 0, 173, 0, 16, 3, 0, 0, 100, 1, 18, 0] payload_rows_per_dst=[8176, 8094, 8190, 2934, 16, 5848, 566, 254] dup_rows_per_dst=[13706, 3492, 13904, 0, 4, 172, 158, 22] combine_rows_per_src=[8176, 8094, 8190, 2934, 16, 5848, 566, 254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61596 payload_rows_remote=53422 dup_rows=3940 dup_rows_remote=3868 dedup_ratio=0.060120 router_tokens_per_expert=[1, 0, 20, 0, 3942, 0, 4090, 0, 0, 0, 0, 4092, 0, 1, 0, 3988, 4086, 292, 8, 2507, 0, 0, 0, 0, 0, 0, 0, 34, 0, 3, 0, 0, 1, 48, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3839, 0, 0, 4077, 48, 17, 69, 64, 39, 0, 305, 42, 0, 2, 0, 0, 36, 0] payload_rows_per_dst=[7924, 7976, 5514, 8174, 8188, 7678, 7950, 8192] dup_rows_per_dst=[2, 2, 456, 72, 2314, 10, 322, 762] combine_rows_per_src=[7924, 7976, 5514, 8174, 8188, 7678, 7950, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60446 payload_rows_remote=52256 dup_rows=5090 dup_rows_remote=4842 dedup_ratio=0.077667 router_tokens_per_expert=[1, 12, 0, 0, 3902, 0, 4094, 3, 2, 2, 32, 4087, 0, 8, 0, 4096, 4040, 0, 42, 2461, 0, 0, 1, 0, 0, 0, 0, 908, 10, 21, 0, 0, 0, 100, 2, 3, 0, 20, 0, 0, 0, 8, 0, 0, 0, 0, 3490, 0, 137, 3642, 26, 560, 0, 365, 46, 1, 134, 0, 0, 0, 480, 0, 22, 10] payload_rows_per_dst=[7832, 8192, 4960, 8084, 8190, 6996, 8000, 8192] dup_rows_per_dst=[4, 88, 48, 1874, 248, 0, 1554, 1274] combine_rows_per_src=[7832, 8192, 4960, 8084, 8190, 6996, 8000, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60838 payload_rows_remote=52656 dup_rows=4698 dup_rows_remote=3976 dedup_ratio=0.071686 router_tokens_per_expert=[0, 0, 0, 1, 3943, 0, 4016, 0, 1, 0, 2, 4056, 0, 0, 4, 3356, 4094, 4, 261, 122, 0, 0, 0, 1, 2, 0, 0, 1335, 0, 325, 0, 0, 99, 7, 57, 0, 0, 0, 7, 0, 6, 0, 1, 8, 90, 16, 3719, 0, 0, 4064, 11, 0, 123, 59, 2957, 2, 0, 0, 0, 0, 6, 0, 12, 1] payload_rows_per_dst=[7888, 6726, 8182, 8188, 8152, 7616, 5948, 8138] dup_rows_per_dst=[0, 0, 722, 3324, 220, 64, 356, 12] combine_rows_per_src=[7888, 6726, 8182, 8188, 8152, 7616, 5948, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56048 payload_rows_remote=50988 dup_rows=9488 dup_rows_remote=9474 dedup_ratio=0.144775 router_tokens_per_expert=[0, 0, 33, 0, 3465, 0, 4076, 3, 8, 0, 29, 4061, 0, 0, 0, 3511, 4047, 101, 5, 2041, 1, 0, 0, 0, 0, 0, 1, 11, 11, 2, 0, 10, 0, 46, 1969, 0, 28, 0, 0, 1, 0, 0, 0, 11, 12, 0, 2514, 0, 37, 4086, 89, 27, 0, 774, 642, 0, 851, 27, 0, 3, 45, 60, 130, 0] payload_rows_per_dst=[6992, 7082, 4240, 8126, 8184, 5060, 8192, 8172] dup_rows_per_dst=[10, 14, 56, 38, 4056, 14, 3118, 2182] combine_rows_per_src=[6992, 7082, 4240, 8126, 8184, 5060, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54370 payload_rows_remote=46198 dup_rows=11166 dup_rows_remote=9746 dedup_ratio=0.170380 router_tokens_per_expert=[0, 0, 0, 0, 3767, 0, 3988, 0, 11, 3, 1, 4090, 0, 0, 0, 4091, 4091, 207, 3, 568, 0, 0, 0, 0, 1, 0, 0, 568, 131, 11, 0, 0, 3, 0, 890, 0, 7, 0, 0, 1, 0, 0, 2, 0, 293, 0, 4077, 0, 0, 4038, 3, 2, 0, 323, 430, 0, 1139, 0, 4, 1, 4, 1, 11, 8] payload_rows_per_dst=[3720, 8188, 1556, 8190, 8192, 8162, 8172, 8190] dup_rows_per_dst=[0, 24, 0, 1414, 1586, 4396, 1420, 2326] combine_rows_per_src=[3720, 8188, 1556, 8190, 8192, 8162, 8172, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=106094592 dispatch_meta_bytes=174616 combine_payload_bytes=106094592 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=106269208 ingress_bytes=181592064 link_bytes=181592064 token_bytes=212363800 weight_movement_bytes=75497472 total_nvlink_bytes=287861272 weight_to_token_ratio=0.355510 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.381705789e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51146 payload_rows_remote=42962 dup_rows=14390 dup_rows_remote=14262 dedup_ratio=0.219574 router_tokens_per_expert=[0, 0, 0, 0, 4001, 0, 4088, 16, 18, 18, 2, 4090, 0, 4, 0, 3678, 4080, 665, 0, 849, 0, 0, 0, 0, 11, 0, 5, 1, 528, 729, 0, 0, 20, 520, 187, 5, 0, 0, 61, 2, 0, 0, 1, 14, 83, 135, 1817, 1, 10, 3907, 556, 1500, 0, 1037, 57, 6, 45, 1, 0, 0, 0, 0, 4, 16] payload_rows_per_dst=[32, 7398, 2948, 8192, 8192, 8128, 8072, 8184] dup_rows_per_dst=[0, 42, 80, 2516, 1574, 3976, 6074, 128] combine_rows_per_src=[32, 7398, 2948, 8192, 8192, 8128, 8072, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=218816512 dispatch_meta_bytes=229160 combine_payload_bytes=218816512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=219045672 ingress_bytes=243982336 link_bytes=243982336 token_bytes=437862184 weight_movement_bytes=25165824 total_nvlink_bytes=463028008 weight_to_token_ratio=0.057474 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25216 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.856423671e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=214040576 dispatch_meta_bytes=228392 combine_payload_bytes=214040576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=214268968 ingress_bytes=239206400 link_bytes=239206400 token_bytes=428309544 weight_movement_bytes=25165824 total_nvlink_bytes=453475368 weight_to_token_ratio=0.058756 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.820084316e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=215678976 dispatch_meta_bytes=226528 combine_payload_bytes=215678976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=215905504 ingress_bytes=266010624 link_bytes=266010624 token_bytes=431584480 weight_movement_bytes=50331648 total_nvlink_bytes=481916128 weight_to_token_ratio=0.116621 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=12544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.024033490e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54936 payload_rows_remote=46784 dup_rows=10600 dup_rows_remote=10534 dedup_ratio=0.161743 router_tokens_per_expert=[7, 0, 75, 0, 3589, 0, 4071, 0, 5, 0, 0, 4092, 0, 0, 0, 4071, 3954, 33, 168, 4, 0, 0, 0, 0, 0, 0, 0, 1399, 0, 124, 0, 20, 2, 63, 0, 0, 22, 0, 6, 0, 0, 0, 0, 0, 385, 1, 3322, 0, 5, 3843, 191, 982, 0, 52, 1587, 0, 693, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[8176, 8152, 8120, 4038, 6298, 6698, 5296, 8158] dup_rows_per_dst=[1074, 66, 6560, 372, 122, 718, 338, 1350] combine_rows_per_src=[8176, 8152, 8120, 4038, 6298, 6698, 5296, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=208846848 dispatch_meta_bytes=241848 combine_payload_bytes=208846848 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=209088696 ingress_bytes=234012672 link_bytes=234012672 token_bytes=417935544 weight_movement_bytes=25165824 total_nvlink_bytes=443101368 weight_to_token_ratio=0.060215 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.780566047e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=189227008 dispatch_meta_bytes=223776 combine_payload_bytes=189227008 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=189450784 ingress_bytes=239558656 link_bytes=239558656 token_bytes=378677792 weight_movement_bytes=50331648 total_nvlink_bytes=429009440 weight_to_token_ratio=0.132914 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=19584 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.822764577e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=175972352 dispatch_meta_bytes=228896 combine_payload_bytes=175972352 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=176201248 ingress_bytes=201138176 link_bytes=201138176 token_bytes=352173600 weight_movement_bytes=25165824 total_nvlink_bytes=377339424 weight_to_token_ratio=0.071459 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=28544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.530429117e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=191627264 dispatch_meta_bytes=229272 combine_payload_bytes=191627264 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=191856536 ingress_bytes=267124736 link_bytes=267124736 token_bytes=383483800 weight_movement_bytes=75497472 total_nvlink_bytes=458981272 weight_to_token_ratio=0.196873 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.032510595e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54146 payload_rows_remote=45966 dup_rows=11390 dup_rows_remote=4774 dedup_ratio=0.173798 router_tokens_per_expert=[0, 0, 23, 0, 0, 4042, 0, 0, 0, 4095, 2477, 0, 0, 85, 0, 665, 0, 0, 5, 2, 0, 0, 1, 6, 0, 0, 2, 0, 9, 0, 0, 4084, 0, 0, 1201, 0, 0, 0, 0, 0, 0, 7, 4064, 4077, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 3804, 0, 0, 5, 0, 4094, 0, 0, 8, 0] payload_rows_per_dst=[8180, 7932, 4460, 8100, 8152, 1502, 7630, 8190] dup_rows_per_dst=[6616, 2356, 12, 2, 2378, 2, 0, 24] combine_rows_per_src=[8180, 7932, 4460, 8100, 8152, 1502, 7630, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64006 payload_rows_remote=55814 dup_rows=1530 dup_rows_remote=1530 dedup_ratio=0.023346 router_tokens_per_expert=[0, 0, 0, 0, 0, 3434, 0, 0, 0, 4088, 3621, 0, 0, 698, 0, 302, 0, 1, 191, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4086, 4092, 0, 0, 0, 0, 0, 14, 0, 0, 6, 0, 3994, 0, 0, 0, 0, 4080, 32, 0, 0, 0] payload_rows_per_dst=[6868, 8182, 8192, 8192, 8182, 8184, 8020, 8186] dup_rows_per_dst=[0, 1060, 404, 0, 20, 0, 8, 38] combine_rows_per_src=[6868, 8182, 8192, 8192, 8182, 8184, 8020, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63524 payload_rows_remote=55412 dup_rows=2012 dup_rows_remote=2008 dedup_ratio=0.030701 router_tokens_per_expert=[19, 0, 0, 0, 0, 4090, 0, 0, 8, 4074, 1801, 1, 0, 0, 0, 1823, 4, 0, 870, 0, 9, 0, 0, 1, 0, 0, 4, 7, 3, 85, 0, 4077, 0, 4, 0, 0, 0, 2, 0, 0, 2, 0, 4052, 3640, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 0, 0, 0, 4093, 3, 0, 0, 0] payload_rows_per_dst=[8192, 7182, 8184, 8192, 8112, 7284, 8188, 8190] dup_rows_per_dst=[26, 84, 1732, 160, 4, 0, 4, 2] combine_rows_per_src=[8192, 7182, 8184, 8192, 8112, 7284, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=188276736 dispatch_meta_bytes=202960 combine_payload_bytes=188276736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188479696 ingress_bytes=213442560 link_bytes=213442560 token_bytes=376756432 weight_movement_bytes=25165824 total_nvlink_bytes=401922256 weight_to_token_ratio=0.066796 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.624051262e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63566 payload_rows_remote=55378 dup_rows=1970 dup_rows_remote=1306 dedup_ratio=0.030060 router_tokens_per_expert=[0, 0, 0, 0, 0, 4089, 0, 0, 1, 4094, 3210, 1, 0, 3, 0, 79, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4060, 28, 0, 492, 0, 1, 0, 0, 0, 0, 1, 3959, 4085, 0, 3, 0, 0, 8, 88, 7, 0, 148, 3, 4095, 0, 3, 0, 0, 3959, 0, 0, 0, 8] payload_rows_per_dst=[8178, 6588, 8188, 8122, 8186, 8178, 8192, 7934] dup_rows_per_dst=[0, 0, 664, 20, 774, 0, 506, 6] combine_rows_per_src=[8178, 6588, 8188, 8122, 8186, 8178, 8192, 7934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63884 payload_rows_remote=55730 dup_rows=1652 dup_rows_remote=1620 dedup_ratio=0.025208 router_tokens_per_expert=[0, 0, 40, 0, 0, 3699, 0, 0, 0, 4090, 1323, 0, 0, 210, 0, 2397, 0, 0, 3, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 19, 717, 0, 4, 1, 0, 0, 0, 0, 4068, 4076, 3, 0, 0, 14, 0, 0, 4, 0, 1, 0, 4094, 0, 0, 0, 0, 3906, 0, 22, 0, 0] payload_rows_per_dst=[7410, 7756, 8192, 8132, 8192, 8154, 8192, 7856] dup_rows_per_dst=[68, 104, 16, 0, 1426, 32, 6, 0] combine_rows_per_src=[7410, 7756, 8192, 8132, 8192, 8154, 8192, 7856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51692 payload_rows_remote=43538 dup_rows=13844 dup_rows_remote=13390 dedup_ratio=0.211243 router_tokens_per_expert=[6, 0, 3, 0, 0, 1822, 0, 0, 0, 4095, 3947, 0, 0, 1061, 0, 568, 0, 0, 610, 8, 0, 0, 3, 0, 0, 0, 12, 0, 1, 0, 0, 4094, 0, 1, 250, 0, 0, 1, 0, 0, 20, 0, 4088, 3940, 1, 0, 0, 107, 5, 2, 7, 0, 9, 0, 4030, 0, 0, 0, 2, 4075, 0, 0, 0, 0] payload_rows_per_dst=[3662, 5594, 8192, 8190, 1816, 7996, 8088, 8154] dup_rows_per_dst=[0, 2020, 1240, 24, 460, 7826, 1820, 454] combine_rows_per_src=[3662, 5594, 8192, 8190, 1816, 7996, 8088, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65248 payload_rows_remote=57286 dup_rows=288 dup_rows_remote=286 dedup_ratio=0.004395 router_tokens_per_expert=[0, 0, 9, 0, 0, 4089, 6, 2, 2, 4093, 3874, 0, 0, 9, 0, 283, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4082, 0, 0, 23, 0, 1, 0, 0, 17, 12, 0, 4091, 4093, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 3974, 0, 0, 1, 0, 4091, 3, 0, 1, 0] payload_rows_per_dst=[8184, 8178, 8188, 8166, 8192, 8192, 7962, 8186] dup_rows_per_dst=[28, 158, 4, 0, 72, 18, 2, 6] combine_rows_per_src=[8184, 8178, 8188, 8166, 8192, 8192, 7962, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=228614144 dispatch_meta_bytes=229376 combine_payload_bytes=228614144 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=228843520 ingress_bytes=253779968 link_bytes=253779968 token_bytes=457457664 weight_movement_bytes=25165824 total_nvlink_bytes=482623488 weight_to_token_ratio=0.055012 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.930972330e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=226967552 dispatch_meta_bytes=229680 combine_payload_bytes=226967552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=227197232 ingress_bytes=252133376 link_bytes=252133376 token_bytes=454164784 weight_movement_bytes=25165824 total_nvlink_bytes=479330608 weight_to_token_ratio=0.055411 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.918443667e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63292 payload_rows_remote=55654 dup_rows=2244 dup_rows_remote=1540 dedup_ratio=0.034241 router_tokens_per_expert=[0, 0, 0, 0, 0, 4062, 0, 0, 0, 3904, 3758, 0, 0, 397, 0, 16, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 16, 0, 4096, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 3453, 0, 0, 0, 0, 4096, 59, 0, 0, 0] payload_rows_per_dst=[8124, 7638, 7868, 8192, 8192, 8168, 6918, 8192] dup_rows_per_dst=[0, 704, 0, 44, 1378, 0, 0, 118] combine_rows_per_src=[8124, 7638, 7868, 8192, 8192, 8168, 6918, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=228270080 dispatch_meta_bytes=229400 combine_payload_bytes=228270080 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=228499480 ingress_bytes=303767552 link_bytes=303767552 token_bytes=456769560 weight_movement_bytes=75497472 total_nvlink_bytes=532267032 weight_to_token_ratio=0.165286 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.311320087e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=226828288 dispatch_meta_bytes=226736 combine_payload_bytes=226828288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=227055024 ingress_bytes=251994112 link_bytes=251994112 token_bytes=453883312 weight_movement_bytes=25165824 total_nvlink_bytes=479049136 weight_to_token_ratio=0.055446 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.917384029e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=178331648 dispatch_meta_bytes=227712 combine_payload_bytes=178331648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=178559360 ingress_bytes=203497472 link_bytes=203497472 token_bytes=356891008 weight_movement_bytes=25165824 total_nvlink_bytes=382056832 weight_to_token_ratio=0.070514 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.548380634e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=234643456 dispatch_meta_bytes=230288 combine_payload_bytes=234643456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234873744 ingress_bytes=259809280 link_bytes=259809280 token_bytes=469517200 weight_movement_bytes=25165824 total_nvlink_bytes=494683024 weight_to_token_ratio=0.053599 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.976848428e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=227958784 dispatch_meta_bytes=228776 combine_payload_bytes=227958784 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=228187560 ingress_bytes=353787904 link_bytes=353787904 token_bytes=456146344 weight_movement_bytes=125829120 total_nvlink_bytes=581975464 weight_to_token_ratio=0.275853 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.691917170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55420 payload_rows_remote=47230 dup_rows=10116 dup_rows_remote=10096 dedup_ratio=0.154358 router_tokens_per_expert=[8, 0, 0, 0, 1, 10, 0, 0, 4086, 0, 0, 0, 141, 0, 0, 4, 3802, 0, 0, 0, 0, 0, 4094, 2, 12, 1, 5, 0, 0, 0, 0, 213, 0, 0, 52, 2, 0, 0, 0, 0, 0, 0, 3954, 4084, 0, 0, 42, 0, 0, 0, 0, 0, 3756, 0, 4086, 2, 3959, 0, 374, 0, 0, 0, 0, 78] payload_rows_per_dst=[8190, 8180, 8188, 6304, 8192, 7946, 7516, 904] dup_rows_per_dst=[20, 282, 9644, 40, 84, 46, 0, 0] combine_rows_per_src=[8190, 8180, 8188, 6304, 8192, 7946, 7516, 904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60172 payload_rows_remote=51982 dup_rows=5364 dup_rows_remote=5350 dedup_ratio=0.081848 router_tokens_per_expert=[0, 0, 0, 0, 87, 1, 0, 192, 4069, 0, 0, 0, 188, 0, 0, 0, 1654, 0, 0, 4, 0, 14, 4015, 104, 0, 0, 0, 0, 31, 0, 0, 129, 0, 1, 6, 1, 1, 0, 0, 0, 0, 2, 4086, 4093, 1, 0, 44, 7, 0, 0, 0, 1, 3938, 0, 4092, 1, 3411, 0, 2099, 1, 0, 0, 0, 495] payload_rows_per_dst=[8192, 8144, 3510, 8110, 8190, 8188, 7880, 7958] dup_rows_per_dst=[552, 370, 42, 240, 14, 92, 0, 4054] combine_rows_per_src=[8192, 8144, 3510, 8110, 8190, 8188, 7880, 7958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65366 payload_rows_remote=57180 dup_rows=170 dup_rows_remote=76 dedup_ratio=0.002594 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 8, 4089, 0, 0, 1, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 5, 0, 0, 0, 53, 0, 0, 5, 0, 0, 0, 0, 0, 0, 11, 4095, 4091, 0, 8, 3, 0, 0, 0, 0, 2, 4089, 0, 4084, 0, 2544, 0, 1506, 0, 0, 1, 0, 1] payload_rows_per_dst=[8168, 8180, 8176, 8186, 8192, 8190, 8180, 8094] dup_rows_per_dst=[18, 2, 0, 94, 0, 44, 2, 10] combine_rows_per_src=[8168, 8180, 8176, 8186, 8192, 8190, 8180, 8094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=193454080 dispatch_meta_bytes=229304 combine_payload_bytes=193454080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193683384 ingress_bytes=218619904 link_bytes=218619904 token_bytes=387137464 weight_movement_bytes=25165824 total_nvlink_bytes=412303288 weight_to_token_ratio=0.065005 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.663444868e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63878 payload_rows_remote=55698 dup_rows=1658 dup_rows_remote=1622 dedup_ratio=0.025299 router_tokens_per_expert=[0, 0, 0, 11, 0, 2, 0, 2, 4091, 0, 0, 0, 1, 0, 0, 0, 4026, 3, 1, 0, 0, 0, 4076, 32, 1, 0, 0, 0, 0, 0, 0, 158, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4087, 4094, 0, 2, 19, 0, 0, 0, 0, 0, 3454, 0, 4030, 0, 3652, 0, 984, 0, 0, 12, 0, 25] payload_rows_per_dst=[8082, 8182, 8060, 8164, 8192, 8180, 6908, 8110] dup_rows_per_dst=[8, 2, 64, 306, 6, 36, 0, 1236] combine_rows_per_src=[8082, 8182, 8060, 8164, 8192, 8180, 6908, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62168 payload_rows_remote=54060 dup_rows=3368 dup_rows_remote=3260 dedup_ratio=0.051392 router_tokens_per_expert=[1, 0, 0, 0, 442, 4, 7, 17, 4093, 0, 0, 0, 4, 0, 0, 0, 4030, 0, 0, 0, 6, 0, 3928, 72, 0, 0, 0, 0, 0, 0, 1, 617, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4095, 10, 36, 528, 0, 2, 32, 0, 0, 4062, 0, 4051, 0, 1534, 0, 1098, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8188, 8108, 7862, 8190, 8192, 8186, 5256] dup_rows_per_dst=[858, 6, 108, 1230, 0, 1152, 6, 8] combine_rows_per_src=[8186, 8188, 8108, 7862, 8190, 8192, 8186, 5256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54850 payload_rows_remote=47310 dup_rows=10686 dup_rows_remote=6378 dedup_ratio=0.163055 router_tokens_per_expert=[0, 5, 0, 24, 0, 8, 0, 45, 4089, 0, 3, 0, 10, 0, 0, 14, 1840, 0, 0, 0, 0, 0, 4082, 18, 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4089, 0, 0, 2056, 0, 0, 0, 0, 0, 4094, 1, 4004, 0, 3606, 0, 512, 0, 0, 0, 0, 179] payload_rows_per_dst=[6746, 7580, 3716, 8178, 8086, 4816, 8188, 7540] dup_rows_per_dst=[116, 40, 0, 6, 0, 2218, 3998, 4308] combine_rows_per_src=[6746, 7580, 3716, 8178, 8086, 4816, 8188, 7540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60874 payload_rows_remote=54416 dup_rows=4662 dup_rows_remote=4640 dedup_ratio=0.071136 router_tokens_per_expert=[0, 2, 0, 13, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 12, 0, 0, 0, 3228, 0, 4087, 0, 4059, 0, 2338, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 5300, 8180, 8192, 8176, 6458, 8190] dup_rows_per_dst=[16, 0, 0, 14, 6, 0, 22, 4604] combine_rows_per_src=[8192, 8186, 5300, 8180, 8192, 8176, 6458, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=212918272 dispatch_meta_bytes=229328 combine_payload_bytes=212918272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213147600 ingress_bytes=238084096 link_bytes=238084096 token_bytes=426065872 weight_movement_bytes=25165824 total_nvlink_bytes=451231696 weight_to_token_ratio=0.059066 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.811544880e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=234209280 dispatch_meta_bytes=229024 combine_payload_bytes=234209280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234438304 ingress_bytes=259375104 link_bytes=259375104 token_bytes=468647584 weight_movement_bytes=25165824 total_nvlink_bytes=493813408 weight_to_token_ratio=0.053699 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.973544850e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62344 payload_rows_remote=54152 dup_rows=3192 dup_rows_remote=3190 dedup_ratio=0.048706 router_tokens_per_expert=[0, 0, 0, 0, 8, 0, 0, 6, 4096, 0, 1, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 4064, 3, 0, 0, 0, 9, 237, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 27, 2, 0, 0, 0, 0, 4096, 0, 4082, 0, 1743, 0, 2202, 0, 0, 0, 0, 26] payload_rows_per_dst=[8172, 8192, 7962, 8192, 8192, 8192, 8192, 5250] dup_rows_per_dst=[20, 2, 2658, 430, 0, 68, 0, 14] combine_rows_per_src=[8172, 8192, 7962, 8192, 8192, 8192, 8192, 5250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=228139008 dispatch_meta_bytes=229280 combine_payload_bytes=228139008 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=228368288 ingress_bytes=328802304 link_bytes=328802304 token_bytes=456507296 weight_movement_bytes=100663296 total_nvlink_bytes=557170592 weight_to_token_ratio=0.220508 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.501805623e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=193781760 dispatch_meta_bytes=214752 combine_payload_bytes=193781760 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=193996512 ingress_bytes=244113408 link_bytes=244113408 token_bytes=387778272 weight_movement_bytes=50331648 total_nvlink_bytes=438109920 weight_to_token_ratio=0.129795 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.857420977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=221429760 dispatch_meta_bytes=229280 combine_payload_bytes=221429760 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=221659040 ingress_bytes=271761408 link_bytes=271761408 token_bytes=443088800 weight_movement_bytes=50331648 total_nvlink_bytes=493420448 weight_to_token_ratio=0.113593 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.067790312e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222887936 dispatch_meta_bytes=236224 combine_payload_bytes=222887936 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=223124160 ingress_bytes=273219584 link_bytes=273219584 token_bytes=446012096 weight_movement_bytes=50331648 total_nvlink_bytes=496343744 weight_to_token_ratio=0.112848 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.078885346e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51548 payload_rows_remote=43358 dup_rows=13988 dup_rows_remote=4826 dedup_ratio=0.213440 router_tokens_per_expert=[0, 0, 4086, 3115, 11, 1204, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 2377, 2657, 4075, 16, 0, 0, 164, 0, 0, 3130, 0, 0, 0, 0, 0, 4096, 41, 8, 2, 0, 2, 23, 3122, 0, 1, 0, 0, 3980, 0, 1, 0, 0, 0, 1, 0, 0, 26, 0, 0, 0, 2] payload_rows_per_dst=[8190, 8192, 7770, 5780, 82, 5888, 7960, 7686] dup_rows_per_dst=[9162, 1236, 3264, 302, 0, 20, 2, 2] combine_rows_per_src=[8190, 8192, 7770, 5780, 82, 5888, 7960, 7686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=221806592 dispatch_meta_bytes=229368 combine_payload_bytes=221806592 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=222035960 ingress_bytes=246972416 link_bytes=246972416 token_bytes=443842552 weight_movement_bytes=25165824 total_nvlink_bytes=469008376 weight_to_token_ratio=0.056700 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.879174725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57300 payload_rows_remote=49108 dup_rows=8236 dup_rows_remote=7308 dedup_ratio=0.125671 router_tokens_per_expert=[5, 0, 4091, 1317, 1, 22, 0, 1, 0, 0, 0, 3854, 1, 13, 4, 1, 0, 29, 0, 7, 0, 0, 27, 2924, 3440, 4089, 0, 0, 7, 0, 5, 0, 445, 0, 0, 0, 19, 0, 4096, 0, 0, 5, 0, 0, 0, 4086, 0, 0, 5, 0, 4082, 0, 0, 0, 0, 0, 191, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2690, 7742, 5948, 8192, 8192, 8180, 8174, 8182] dup_rows_per_dst=[2, 4, 26, 6890, 928, 2, 0, 384] combine_rows_per_src=[2690, 7742, 5948, 8192, 8192, 8180, 8174, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55802 payload_rows_remote=47610 dup_rows=9734 dup_rows_remote=6404 dedup_ratio=0.148529 router_tokens_per_expert=[0, 0, 4096, 3293, 0, 2, 0, 0, 0, 0, 0, 1481, 0, 0, 0, 0, 0, 17, 0, 1, 0, 0, 209, 2329, 1684, 4076, 0, 0, 0, 0, 0, 1, 4007, 0, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 0, 10, 3931, 0, 20, 0, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0] payload_rows_per_dst=[6590, 4940, 4956, 8192, 8092, 7864, 6976, 8192] dup_rows_per_dst=[0, 2522, 156, 3330, 3600, 58, 0, 68] combine_rows_per_src=[6590, 4940, 4956, 8192, 8092, 7864, 6976, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=177594368 dispatch_meta_bytes=192736 combine_payload_bytes=177594368 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=177787104 ingress_bytes=227926016 link_bytes=227926016 token_bytes=355381472 weight_movement_bytes=50331648 total_nvlink_bytes=405713120 weight_to_token_ratio=0.141627 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.734253628e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56114 payload_rows_remote=47926 dup_rows=9422 dup_rows_remote=9024 dedup_ratio=0.143768 router_tokens_per_expert=[0, 0, 4092, 3570, 0, 1, 0, 0, 0, 0, 0, 493, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 2, 3648, 3578, 4094, 2, 0, 0, 4, 0, 0, 796, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 3, 4094, 0, 196, 3, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0] payload_rows_per_dst=[7140, 986, 7362, 8192, 8080, 8188, 7976, 8190] dup_rows_per_dst=[2, 0, 336, 7164, 1514, 398, 2, 6] combine_rows_per_src=[7140, 986, 7362, 8192, 8080, 8188, 7976, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55440 payload_rows_remote=47254 dup_rows=10096 dup_rows_remote=10088 dedup_ratio=0.154053 router_tokens_per_expert=[0, 0, 4086, 2137, 0, 2, 0, 0, 0, 0, 0, 4009, 0, 63, 1, 0, 0, 141, 0, 0, 0, 0, 0, 2804, 212, 4053, 0, 0, 0, 148, 0, 0, 2645, 0, 0, 0, 7, 0, 4085, 8, 0, 0, 0, 0, 157, 4093, 0, 2, 9, 0, 4095, 0, 0, 0, 0, 0, 2, 7, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[4278, 8050, 5654, 4708, 8186, 8186, 8192, 8186] dup_rows_per_dst=[0, 96, 236, 220, 6510, 318, 2708, 8] combine_rows_per_src=[4278, 8050, 5654, 4708, 8186, 8186, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53904 payload_rows_remote=46344 dup_rows=11632 dup_rows_remote=10278 dedup_ratio=0.177490 router_tokens_per_expert=[0, 0, 4093, 791, 0, 733, 2, 0, 0, 3, 2, 2979, 0, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 3605, 24, 3946, 0, 0, 5, 8, 0, 0, 3456, 0, 0, 5, 0, 48, 4085, 0, 0, 0, 0, 0, 8, 3911, 0, 1168, 0, 0, 3857, 0, 0, 3, 0, 0, 1, 2, 0, 14, 0, 0, 0, 9] payload_rows_per_dst=[6352, 8182, 7560, 2602, 5326, 7970, 7720, 8192] dup_rows_per_dst=[2016, 5956, 1354, 48, 8, 2204, 0, 46] combine_rows_per_src=[6352, 8182, 7560, 2602, 5326, 7970, 7720, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57406 payload_rows_remote=49770 dup_rows=8130 dup_rows_remote=8130 dedup_ratio=0.124054 router_tokens_per_expert=[0, 0, 4096, 3946, 0, 0, 0, 0, 0, 0, 1, 725, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 3964, 634, 4084, 13, 0, 0, 33, 0, 0, 3379, 0, 0, 0, 5, 0, 4086, 2, 0, 0, 0, 0, 0, 3964, 0, 4, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0] payload_rows_per_dst=[7892, 1452, 7932, 8192, 8176, 7934, 7636, 8192] dup_rows_per_dst=[0, 0, 14, 1336, 6768, 2, 0, 10] combine_rows_per_src=[7892, 1452, 7932, 8192, 8176, 7934, 7636, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49574 payload_rows_remote=41382 dup_rows=15962 dup_rows_remote=7976 dedup_ratio=0.243561 router_tokens_per_expert=[0, 0, 4096, 1518, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4, 0, 0, 0, 1180, 0, 17, 0, 0, 43, 3425, 170, 4093, 2, 0, 0, 22, 0, 0, 1819, 0, 0, 0, 35, 2, 4010, 0, 0, 0, 0, 0, 0, 4096, 0, 66, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8162, 388, 72, 8192, 8190, 8192] dup_rows_per_dst=[3036, 7986, 4806, 0, 2, 132, 0, 0] combine_rows_per_src=[8186, 8192, 8162, 388, 72, 8192, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=201146368 dispatch_meta_bytes=225664 combine_payload_bytes=201146368 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=201372032 ingress_bytes=276643840 link_bytes=276643840 token_bytes=402518400 weight_movement_bytes=75497472 total_nvlink_bytes=478015872 weight_to_token_ratio=0.187563 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18176 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.104939978e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=195010560 dispatch_meta_bytes=216056 combine_payload_bytes=195010560 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=195226616 ingress_bytes=295673856 link_bytes=295673856 token_bytes=390237176 weight_movement_bytes=100663296 total_nvlink_bytes=490900472 weight_to_token_ratio=0.257954 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.249736412e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=196304896 dispatch_meta_bytes=227800 combine_payload_bytes=196304896 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=196532696 ingress_bytes=221470720 link_bytes=221470720 token_bytes=392837592 weight_movement_bytes=25165824 total_nvlink_bytes=418003416 weight_to_token_ratio=0.064062 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.685136284e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=203857920 dispatch_meta_bytes=231600 combine_payload_bytes=203857920 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=204089520 ingress_bytes=229023744 link_bytes=229023744 token_bytes=407947440 weight_movement_bytes=25165824 total_nvlink_bytes=433113264 weight_to_token_ratio=0.061689 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.742606070e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53396 payload_rows_remote=45204 dup_rows=12140 dup_rows_remote=3060 dedup_ratio=0.185242 router_tokens_per_expert=[4046, 18, 2797, 1, 265, 4042, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 24, 1, 14, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1343, 20, 111, 0, 683, 0, 0, 0, 1, 4085, 0, 0, 0, 0, 0, 4087, 0, 3756, 3, 29, 2, 2, 0, 0, 0, 0, 4075, 0, 0, 560, 0, 0, 7, 0, 0, 1, 0] payload_rows_per_dst=[8192, 5606, 7816, 3418, 5488, 8182, 8124, 6570] dup_rows_per_dst=[9080, 12, 14, 896, 0, 2124, 0, 14] combine_rows_per_src=[8192, 5606, 7816, 3418, 5488, 8182, 8124, 6570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=193552384 dispatch_meta_bytes=229368 combine_payload_bytes=193552384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193781752 ingress_bytes=218718208 link_bytes=218718208 token_bytes=387334136 weight_movement_bytes=25165824 total_nvlink_bytes=412499960 weight_to_token_ratio=0.064972 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.664192848e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=189825024 dispatch_meta_bytes=226488 combine_payload_bytes=189825024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=190051512 ingress_bytes=214990848 link_bytes=214990848 token_bytes=379876536 weight_movement_bytes=25165824 total_nvlink_bytes=405042360 weight_to_token_ratio=0.066247 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.635831945e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=169500672 dispatch_meta_bytes=197432 combine_payload_bytes=169500672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=169698104 ingress_bytes=194666496 link_bytes=194666496 token_bytes=339198776 weight_movement_bytes=25165824 total_nvlink_bytes=364364600 weight_to_token_ratio=0.074192 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.481187110e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=185155584 dispatch_meta_bytes=193056 combine_payload_bytes=185155584 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=185348640 ingress_bytes=285818880 link_bytes=285818880 token_bytes=370504224 weight_movement_bytes=100663296 total_nvlink_bytes=471167520 weight_to_token_ratio=0.271693 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.174751431e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58898 payload_rows_remote=50732 dup_rows=6638 dup_rows_remote=6620 dedup_ratio=0.101288 router_tokens_per_expert=[3154, 14, 2465, 0, 7, 4077, 0, 0, 0, 0, 0, 4, 2894, 0, 1, 0, 16, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 543, 0, 3284, 1, 310, 1, 0, 0, 2, 4076, 7, 6, 0, 49, 0, 4087, 0, 3612, 0, 0, 0, 0, 4, 0, 0, 11, 4024, 17, 0, 68, 0, 4, 12, 0, 0, 10, 0] payload_rows_per_dst=[4950, 5798, 8186, 8180, 8166, 7322, 8104, 8192] dup_rows_per_dst=[22, 0, 16, 6404, 18, 0, 8, 170] combine_rows_per_src=[4950, 5798, 8186, 8180, 8166, 7322, 8104, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63864 payload_rows_remote=55674 dup_rows=1672 dup_rows_remote=252 dedup_ratio=0.025513 router_tokens_per_expert=[4092, 0, 3346, 0, 248, 4087, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 627, 0, 85, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6954, 7784, 8174, 8190, 8192, 8192, 8186, 8192] dup_rows_per_dst=[234, 0, 0, 1420, 0, 4, 0, 14] combine_rows_per_src=[6954, 7784, 8174, 8190, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65232 payload_rows_remote=57072 dup_rows=304 dup_rows_remote=298 dedup_ratio=0.004639 router_tokens_per_expert=[4060, 0, 4087, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 3994, 3, 3, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 40, 0, 17, 0, 9, 0, 0, 0, 3, 4094, 0, 6, 19, 3, 0, 4076, 0, 4072, 0, 8, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 25, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8000, 8192, 8136, 8192, 8160, 8188, 8188] dup_rows_per_dst=[8, 0, 90, 116, 52, 6, 0, 32] combine_rows_per_src=[8176, 8000, 8192, 8136, 8192, 8160, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59324 payload_rows_remote=51138 dup_rows=6212 dup_rows_remote=6134 dedup_ratio=0.094788 router_tokens_per_expert=[3229, 6, 3968, 0, 927, 4096, 0, 0, 0, 0, 0, 0, 1991, 3, 38, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 147, 0, 871, 9, 0, 0, 0, 4096, 0, 11, 1028, 0, 1, 4095, 0, 4082, 0, 0, 0, 0, 0, 39, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5622, 5380, 8192, 7396, 8192, 8166, 8186, 8190] dup_rows_per_dst=[406, 2458, 4, 1170, 2096, 0, 78, 0] combine_rows_per_src=[5622, 5380, 8192, 7396, 8192, 8166, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55976 payload_rows_remote=47784 dup_rows=9560 dup_rows_remote=8170 dedup_ratio=0.145874 router_tokens_per_expert=[2151, 0, 3315, 0, 2153, 4095, 0, 0, 0, 0, 0, 0, 3805, 75, 0, 0, 0, 0, 0, 0, 0, 16, 196, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 1, 0, 0, 4085, 0, 250, 8, 0, 9, 4080, 0, 3682, 0, 0, 0, 0, 0, 4, 0, 0, 4095, 0, 0, 705, 0, 0, 0, 3, 3, 0, 0] payload_rows_per_dst=[4306, 8170, 8192, 4366, 7178, 7382, 8190, 8192] dup_rows_per_dst=[0, 6220, 422, 10, 490, 1020, 8, 1390] combine_rows_per_src=[4306, 8170, 8192, 4366, 7178, 7382, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=207798272 dispatch_meta_bytes=229408 combine_payload_bytes=207798272 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=208027680 ingress_bytes=283295744 link_bytes=283295744 token_bytes=415825952 weight_movement_bytes=75497472 total_nvlink_bytes=491323424 weight_to_token_ratio=0.181560 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.155553282e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62830 payload_rows_remote=55144 dup_rows=2706 dup_rows_remote=2706 dedup_ratio=0.041290 router_tokens_per_expert=[3958, 0, 4019, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 450, 0, 27, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 4096, 0, 4094, 0, 5, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8136, 7686, 8154, 6122, 8192, 8188, 8160, 8192] dup_rows_per_dst=[2422, 0, 0, 240, 20, 10, 0, 14] combine_rows_per_src=[8136, 7686, 8154, 6122, 8192, 8188, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62046 payload_rows_remote=53984 dup_rows=3490 dup_rows_remote=3468 dedup_ratio=0.053253 router_tokens_per_expert=[3973, 0, 710, 0, 2804, 4020, 0, 0, 0, 0, 0, 0, 4046, 164, 1, 0, 0, 2, 4, 0, 0, 7, 0, 9, 0, 1, 0, 0, 0, 544, 0, 218, 0, 0, 0, 0, 4088, 0, 0, 542, 0, 3, 4090, 0, 3314, 0, 73, 0, 4, 0, 56, 0, 0, 4092, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6580, 8180, 8062, 7962, 8192, 6698, 8192, 8180] dup_rows_per_dst=[448, 242, 22, 1510, 1068, 82, 112, 6] combine_rows_per_src=[6580, 8180, 8062, 7962, 8192, 6698, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=228040704 dispatch_meta_bytes=223704 combine_payload_bytes=228040704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=228264408 ingress_bytes=253206528 link_bytes=253206528 token_bytes=456305112 weight_movement_bytes=25165824 total_nvlink_bytes=481470936 weight_to_token_ratio=0.055151 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=23424 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.926609114e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56302 payload_rows_remote=48126 dup_rows=9234 dup_rows_remote=7832 dedup_ratio=0.140900 router_tokens_per_expert=[0, 1, 0, 4070, 4086, 0, 472, 95, 0, 0, 12, 0, 0, 0, 36, 0, 0, 0, 0, 16, 0, 0, 0, 3873, 0, 0, 0, 44, 0, 0, 4001, 0, 4009, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 105, 1, 1, 0, 0, 33, 3806, 0, 0, 3954, 0, 8, 2, 0, 4096] payload_rows_per_dst=[8176, 8190, 7778, 8172, 8026, 8192, 7750, 18] dup_rows_per_dst=[1402, 36, 0, 7566, 0, 86, 142, 2] combine_rows_per_src=[8176, 8190, 7778, 8172, 8026, 8192, 7750, 18] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=233766912 dispatch_meta_bytes=229480 combine_payload_bytes=233766912 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=233996392 ingress_bytes=284098560 link_bytes=284098560 token_bytes=467763304 weight_movement_bytes=50331648 total_nvlink_bytes=518094952 weight_to_token_ratio=0.107601 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.161661784e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=209461248 dispatch_meta_bytes=229088 combine_payload_bytes=209461248 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=209690336 ingress_bytes=234627072 link_bytes=234627072 token_bytes=419151584 weight_movement_bytes=25165824 total_nvlink_bytes=444317408 weight_to_token_ratio=0.060040 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785240921e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=195723264 dispatch_meta_bytes=223816 combine_payload_bytes=195723264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=195947080 ingress_bytes=220889088 link_bytes=220889088 token_bytes=391670344 weight_movement_bytes=25165824 total_nvlink_bytes=416836168 weight_to_token_ratio=0.064253 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.680710736e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=225869824 dispatch_meta_bytes=231400 combine_payload_bytes=225869824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=226101224 ingress_bytes=251035648 link_bytes=251035648 token_bytes=451971048 weight_movement_bytes=25165824 total_nvlink_bytes=477136872 weight_to_token_ratio=0.055680 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=27264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.910091226e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=221118464 dispatch_meta_bytes=229808 combine_payload_bytes=221118464 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=221348272 ingress_bytes=246284288 link_bytes=246284288 token_bytes=442466736 weight_movement_bytes=25165824 total_nvlink_bytes=467632560 weight_to_token_ratio=0.056876 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.873938866e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=197124096 dispatch_meta_bytes=223832 combine_payload_bytes=197124096 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=197347928 ingress_bytes=322953216 link_bytes=322953216 token_bytes=394472024 weight_movement_bytes=125829120 total_nvlink_bytes=520301144 weight_to_token_ratio=0.318981 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.457300822e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61168 payload_rows_remote=54796 dup_rows=4368 dup_rows_remote=3946 dedup_ratio=0.066650 router_tokens_per_expert=[2, 6, 0, 4061, 4096, 0, 288, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3027, 2, 2943, 0, 0, 232, 0, 193, 18, 11, 0, 0, 119, 3, 18, 0, 0, 1467, 0, 5, 1, 0, 0, 1, 0, 3936, 0, 0, 4090, 4, 1, 0, 85, 4065] payload_rows_per_dst=[8192, 8132, 8178, 6058, 6372, 8166, 7886, 8184] dup_rows_per_dst=[592, 0, 0, 0, 422, 3178, 0, 176] combine_rows_per_src=[8192, 8132, 8178, 6058, 6372, 8166, 7886, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63214 payload_rows_remote=56856 dup_rows=2322 dup_rows_remote=2322 dedup_ratio=0.035431 router_tokens_per_expert=[0, 0, 0, 4093, 4094, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 75, 0, 0, 3104, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 5, 0, 4, 3994, 0, 0, 4089, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8186, 7932, 6358, 8188, 8192, 7988, 8178] dup_rows_per_dst=[734, 0, 0, 0, 2, 1568, 18, 0] combine_rows_per_src=[8192, 8186, 7932, 6358, 8188, 8192, 7988, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62350 payload_rows_remote=54182 dup_rows=3186 dup_rows_remote=2446 dedup_ratio=0.048615 router_tokens_per_expert=[0, 0, 0, 4087, 4096, 0, 0, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 2, 0, 2973, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 99, 178, 0, 7, 0, 0, 1, 0, 0, 3810, 0, 0, 4095, 0, 0, 0, 0, 4072] payload_rows_per_dst=[8192, 8174, 7854, 5946, 8192, 8168, 7634, 8190] dup_rows_per_dst=[2440, 0, 0, 4, 0, 740, 2, 0] combine_rows_per_src=[8192, 8174, 7854, 5946, 8192, 8168, 7634, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=224444416 dispatch_meta_bytes=234968 combine_payload_bytes=224444416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=224679384 ingress_bytes=249610240 link_bytes=249610240 token_bytes=449123800 weight_movement_bytes=25165824 total_nvlink_bytes=474289624 weight_to_token_ratio=0.056033 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.899245518e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59260 payload_rows_remote=51068 dup_rows=6276 dup_rows_remote=6264 dedup_ratio=0.095764 router_tokens_per_expert=[0, 0, 0, 4096, 4082, 0, 110, 529, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4, 0, 4, 3794, 0, 0, 0, 34, 0, 0, 3530, 0, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 0, 8, 0, 0, 0, 18, 207, 0, 5, 0, 0, 0, 0, 0, 4010, 0, 0, 4085, 0, 0, 0, 1, 4096] payload_rows_per_dst=[8184, 8192, 7602, 8182, 8192, 8192, 8030, 2686] dup_rows_per_dst=[1258, 12, 2, 4432, 106, 466, 0, 0] combine_rows_per_src=[8184, 8192, 7602, 8182, 8192, 8192, 8030, 2686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50238 payload_rows_remote=42046 dup_rows=15298 dup_rows_remote=7002 dedup_ratio=0.233429 router_tokens_per_expert=[0, 7, 4, 4059, 4027, 0, 1, 277, 0, 0, 40, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 4041, 0, 0, 0, 16, 11, 0, 3869, 0, 4096, 1, 0, 0, 5, 42, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 4096, 0, 0, 50, 2, 4096] payload_rows_per_dst=[556, 8184, 8106, 7782, 8192, 3130, 6096, 8192] dup_rows_per_dst=[22, 16, 1710, 10, 1476, 0, 3768, 8296] combine_rows_per_src=[556, 8184, 8106, 7782, 8192, 3130, 6096, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57732 payload_rows_remote=49718 dup_rows=7804 dup_rows_remote=7804 dedup_ratio=0.119080 router_tokens_per_expert=[0, 0, 0, 4096, 4092, 0, 8, 13, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 0, 4026, 0, 0, 0, 38, 0, 0, 2689, 0, 4087, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 50, 1447, 0, 0, 0, 0, 0, 0, 1, 4006, 0, 0, 4080, 0, 0, 5, 0, 4096] payload_rows_per_dst=[7820, 8192, 8054, 5454, 8176, 3836, 8014, 8186] dup_rows_per_dst=[34, 28, 372, 0, 0, 1960, 0, 5410] combine_rows_per_src=[7820, 8192, 8054, 5454, 8176, 3836, 8014, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=232882176 dispatch_meta_bytes=236712 combine_payload_bytes=232882176 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233118888 ingress_bytes=258048000 link_bytes=258048000 token_bytes=466001064 weight_movement_bytes=25165824 total_nvlink_bytes=491166888 weight_to_token_ratio=0.054004 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=25856 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.963447122e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62360 payload_rows_remote=54412 dup_rows=3176 dup_rows_remote=3176 dedup_ratio=0.048462 router_tokens_per_expert=[0, 0, 0, 4084, 3943, 0, 0, 928, 0, 0, 5, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 3971, 0, 0, 0, 123, 0, 0, 2663, 0, 4085, 1, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 92, 5, 166, 485, 0, 31, 2, 0, 80, 0, 0, 3991, 0, 0, 3997, 1, 1, 0, 10, 4083] payload_rows_per_dst=[8124, 8190, 7948, 5572, 8184, 8192, 8136, 8014] dup_rows_per_dst=[1618, 4, 0, 0, 8, 1470, 72, 4] combine_rows_per_src=[8124, 8190, 7948, 5572, 8184, 8192, 8136, 8014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=221929472 dispatch_meta_bytes=226512 combine_payload_bytes=221929472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=222155984 ingress_bytes=247095296 link_bytes=247095296 token_bytes=444085456 weight_movement_bytes=25165824 total_nvlink_bytes=469251280 weight_to_token_ratio=0.056669 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.880109700e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56404 payload_rows_remote=48218 dup_rows=9132 dup_rows_remote=956 dedup_ratio=0.139343 router_tokens_per_expert=[0, 0, 0, 4069, 0, 0, 19, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 4093, 0, 0, 4088, 0, 24, 0, 250, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 4095, 0, 0, 0, 1, 1, 4018, 0, 0, 0, 10, 0, 0, 0, 4, 4067, 0, 0, 0, 1, 3894, 10, 0, 11, 0, 1, 0, 0, 0, 1] payload_rows_per_dst=[8186, 8192, 1264, 8188, 6650, 8056, 7678, 8190] dup_rows_per_dst=[8176, 830, 24, 80, 2, 0, 8, 12] combine_rows_per_src=[8186, 8192, 1264, 8188, 6650, 8056, 7678, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=209174528 dispatch_meta_bytes=229328 combine_payload_bytes=209174528 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=209403856 ingress_bytes=234340352 link_bytes=234340352 token_bytes=418578384 weight_movement_bytes=25165824 total_nvlink_bytes=443744208 weight_to_token_ratio=0.060122 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783059313e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=172220416 dispatch_meta_bytes=196192 combine_payload_bytes=172220416 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=172416608 ingress_bytes=247717888 link_bytes=247717888 token_bytes=344637024 weight_movement_bytes=75497472 total_nvlink_bytes=420134496 weight_to_token_ratio=0.219064 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=25856 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.884846905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=203644928 dispatch_meta_bytes=230088 combine_payload_bytes=203644928 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203875016 ingress_bytes=228810752 link_bytes=228810752 token_bytes=407519944 weight_movement_bytes=25165824 total_nvlink_bytes=432685768 weight_to_token_ratio=0.061754 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.740985447e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=222871552 dispatch_meta_bytes=230352 combine_payload_bytes=222871552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=223101904 ingress_bytes=248037376 link_bytes=248037376 token_bytes=445973456 weight_movement_bytes=25165824 total_nvlink_bytes=471139280 weight_to_token_ratio=0.056429 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.887277840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=197500928 dispatch_meta_bytes=196696 combine_payload_bytes=197500928 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=197697624 ingress_bytes=247832576 link_bytes=247832576 token_bytes=395198552 weight_movement_bytes=50331648 total_nvlink_bytes=445530200 weight_to_token_ratio=0.127358 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18048 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.885719549e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58944 payload_rows_remote=50752 dup_rows=6592 dup_rows_remote=6546 dedup_ratio=0.100586 router_tokens_per_expert=[0, 0, 0, 4018, 1, 0, 1476, 20, 0, 0, 0, 0, 4065, 0, 0, 0, 3841, 0, 0, 4057, 2, 98, 0, 226, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 12, 10, 8, 1, 3995, 0, 0, 106, 0, 0, 2, 0, 0, 4033, 0, 36, 16, 0, 2623, 0, 0, 0, 0, 0, 11, 14, 6, 0] payload_rows_per_dst=[4126, 8130, 8054, 8072, 8192, 8150, 6068, 8152] dup_rows_per_dst=[2174, 0, 280, 0, 46, 56, 4012, 24] combine_rows_per_src=[4126, 8130, 8054, 8072, 8192, 8150, 6068, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58772 payload_rows_remote=50588 dup_rows=6764 dup_rows_remote=6764 dedup_ratio=0.103210 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 263, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4039, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 8192, 8184, 1558, 8078] dup_rows_per_dst=[526, 0, 6226, 0, 8, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 8192, 8184, 1558, 8078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56658 payload_rows_remote=48466 dup_rows=8878 dup_rows_remote=8878 dedup_ratio=0.135468 router_tokens_per_expert=[0, 0, 0, 4082, 0, 0, 26, 0, 0, 0, 0, 0, 3748, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 1862, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 2589, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[52, 7496, 8192, 8152, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 3724, 0, 0, 0, 5154, 0] combine_rows_per_src=[52, 7496, 8192, 8152, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=207880192 dispatch_meta_bytes=229192 combine_payload_bytes=207880192 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=208109384 ingress_bytes=283377664 link_bytes=283377664 token_bytes=415989576 weight_movement_bytes=75497472 total_nvlink_bytes=491487048 weight_to_token_ratio=0.181489 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.156176598e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57800 payload_rows_remote=49846 dup_rows=7736 dup_rows_remote=7732 dedup_ratio=0.118042 router_tokens_per_expert=[0, 0, 0, 4069, 0, 0, 308, 0, 0, 26, 0, 0, 3953, 0, 0, 0, 4084, 0, 0, 4093, 0, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 173, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7954, 5756, 8192, 8192, 8190, 3138, 8186] dup_rows_per_dst=[7046, 4, 340, 0, 346, 0, 0, 0] combine_rows_per_src=[8192, 7954, 5756, 8192, 8192, 8190, 3138, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=207208448 dispatch_meta_bytes=229408 combine_payload_bytes=207208448 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=207437856 ingress_bytes=257540096 link_bytes=257540096 token_bytes=414646304 weight_movement_bytes=50331648 total_nvlink_bytes=464977952 weight_to_token_ratio=0.121385 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.959582559e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61088 payload_rows_remote=53386 dup_rows=4448 dup_rows_remote=3668 dedup_ratio=0.067871 router_tokens_per_expert=[0, 0, 0, 4093, 0, 0, 3049, 0, 0, 36, 0, 2, 4096, 0, 0, 0, 3757, 0, 0, 3840, 0, 0, 0, 271, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 1296, 0, 0, 0, 0, 0, 8, 0, 0, 12] payload_rows_per_dst=[6098, 8192, 7144, 8188, 7418, 8154, 8192, 7702] dup_rows_per_dst=[0, 76, 150, 120, 0, 734, 2588, 780] combine_rows_per_src=[6098, 8192, 7144, 8188, 7418, 8154, 8192, 7702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57598 payload_rows_remote=49414 dup_rows=7938 dup_rows_remote=306 dedup_ratio=0.121124 router_tokens_per_expert=[0, 0, 0, 4066, 0, 0, 224, 0, 2, 15, 12, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 4014, 0, 0, 0, 1, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[448, 8192, 8192, 8172, 8190, 8028, 8184, 8192] dup_rows_per_dst=[0, 58, 242, 0, 4, 2, 7632, 0] combine_rows_per_src=[448, 8192, 8192, 8172, 8190, 8028, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59348 payload_rows_remote=51632 dup_rows=6188 dup_rows_remote=6148 dedup_ratio=0.094421 router_tokens_per_expert=[0, 0, 0, 4091, 0, 19, 2048, 2, 1, 0, 13, 0, 4096, 0, 0, 0, 3858, 0, 0, 4015, 0, 0, 0, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 978, 0, 0, 0, 3882, 0, 12, 0, 1, 0, 0, 0, 0, 4093, 6, 19, 0, 0, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7716, 8188, 8156, 7766, 3108, 8030] dup_rows_per_dst=[4128, 28, 40, 0, 1948, 24, 20, 0] combine_rows_per_src=[8192, 8192, 7716, 8188, 8156, 7766, 3108, 8030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=198516736 dispatch_meta_bytes=229376 combine_payload_bytes=198516736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198746112 ingress_bytes=223682560 link_bytes=223682560 token_bytes=397262848 weight_movement_bytes=25165824 total_nvlink_bytes=422428672 weight_to_token_ratio=0.063348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.701965831e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58468 payload_rows_remote=50276 dup_rows=7068 dup_rows_remote=7006 dedup_ratio=0.107849 router_tokens_per_expert=[0, 0, 0, 0, 6, 0, 23, 2, 0, 2, 3963, 0, 0, 4075, 8, 0, 4071, 454, 0, 0, 4096, 6, 0, 0, 4093, 0, 0, 2602, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4, 1, 0, 0, 0, 999, 0, 1, 0, 0, 0, 0, 141, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8174, 7836, 8186, 8158, 7566, 2164, 8192] dup_rows_per_dst=[62, 822, 856, 5204, 0, 6, 118, 0] combine_rows_per_src=[8192, 8174, 7836, 8186, 8158, 7566, 2164, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=204169216 dispatch_meta_bytes=230312 combine_payload_bytes=204169216 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=204399528 ingress_bytes=229335040 link_bytes=229335040 token_bytes=408568744 weight_movement_bytes=25165824 total_nvlink_bytes=433734568 weight_to_token_ratio=0.061595 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.744974672e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=218669056 dispatch_meta_bytes=228216 combine_payload_bytes=218669056 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=218897272 ingress_bytes=269000704 link_bytes=269000704 token_bytes=437566328 weight_movement_bytes=50331648 total_nvlink_bytes=487897976 weight_to_token_ratio=0.115026 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.046784544e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=202399744 dispatch_meta_bytes=198880 combine_payload_bytes=202399744 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=202598624 ingress_bytes=277897216 link_bytes=277897216 token_bytes=404998368 weight_movement_bytes=75497472 total_nvlink_bytes=480495840 weight_to_token_ratio=0.186414 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.114476721e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211484672 dispatch_meta_bytes=231120 combine_payload_bytes=211484672 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=211715792 ingress_bytes=312147968 link_bytes=312147968 token_bytes=423200464 weight_movement_bytes=100663296 total_nvlink_bytes=523863760 weight_to_token_ratio=0.237862 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.375085369e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=205930496 dispatch_meta_bytes=229128 combine_payload_bytes=205930496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206159624 ingress_bytes=231096320 link_bytes=231096320 token_bytes=412090120 weight_movement_bytes=25165824 total_nvlink_bytes=437255944 weight_to_token_ratio=0.061069 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.758375978e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64482 payload_rows_remote=56290 dup_rows=1054 dup_rows_remote=1036 dedup_ratio=0.016083 router_tokens_per_expert=[4, 1, 0, 0, 0, 10, 0, 0, 44, 0, 4068, 0, 0, 4086, 0, 0, 4064, 359, 5, 0, 4038, 20, 0, 0, 4067, 33, 0, 4086, 8, 0, 0, 0, 4088, 7, 0, 0, 10, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 12, 0, 97, 0, 8, 0, 0, 0, 3635, 0, 0, 3, 2, 1, 5, 0] payload_rows_per_dst=[8100, 8186, 8166, 8192, 8192, 8178, 8188, 7280] dup_rows_per_dst=[6, 38, 730, 62, 18, 8, 180, 12] combine_rows_per_src=[8100, 8186, 8166, 8192, 8192, 8178, 8188, 7280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63628 payload_rows_remote=55694 dup_rows=1908 dup_rows_remote=1908 dedup_ratio=0.029114 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 7, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3967, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 939, 0, 0, 0, 0, 0, 0, 1, 3272, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 7934, 8192, 8192, 8192, 6544] dup_rows_per_dst=[0, 0, 14, 0, 0, 14, 1880, 0] combine_rows_per_src=[8192, 8190, 8192, 7934, 8192, 8192, 8192, 6544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63728 payload_rows_remote=55536 dup_rows=1808 dup_rows_remote=1770 dedup_ratio=0.027588 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 6422, 8170, 8192, 8176, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 38, 0, 1770] combine_rows_per_src=[8192, 8192, 8192, 6422, 8170, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230563840 dispatch_meta_bytes=229304 combine_payload_bytes=230563840 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230793144 ingress_bytes=255729664 link_bytes=255729664 token_bytes=461356984 weight_movement_bytes=25165824 total_nvlink_bytes=486522808 weight_to_token_ratio=0.054547 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.945807264e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=228122624 dispatch_meta_bytes=230408 combine_payload_bytes=228122624 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=228353032 ingress_bytes=303620096 link_bytes=303620096 token_bytes=456475656 weight_movement_bytes=75497472 total_nvlink_bytes=531973128 weight_to_token_ratio=0.165392 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.310198117e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56780 payload_rows_remote=49248 dup_rows=8756 dup_rows_remote=8756 dedup_ratio=0.133606 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3766, 0, 0, 4096, 0, 0, 4096, 25, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 15, 6, 0, 0, 0, 642, 1, 0, 0, 0, 0, 0, 45, 3497, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7532, 8192, 8192, 8192, 8192, 1294, 6994] dup_rows_per_dst=[2, 0, 50, 8180, 0, 442, 82, 0] combine_rows_per_src=[8192, 7532, 8192, 8192, 8192, 8192, 1294, 6994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56118 payload_rows_remote=49092 dup_rows=9418 dup_rows_remote=7464 dedup_ratio=0.143707 router_tokens_per_expert=[0, 0, 11, 0, 0, 0, 30, 0, 0, 0, 4054, 0, 0, 4096, 0, 0, 4079, 815, 0, 0, 4095, 1, 0, 0, 4095, 0, 0, 1802, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 51, 0, 0, 0, 2709, 0, 0, 0, 0, 0, 0, 127, 2688, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8142, 8108, 3022, 5244, 8192, 8192, 8192, 7026] dup_rows_per_dst=[80, 0, 1564, 0, 14, 136, 5670, 1954] combine_rows_per_src=[8142, 8108, 3022, 5244, 8192, 8192, 8192, 7026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57264 payload_rows_remote=49072 dup_rows=8272 dup_rows_remote=7030 dedup_ratio=0.126221 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 2, 0, 0, 1, 4057, 0, 1, 4096, 0, 0, 4095, 14, 0, 0, 4096, 129, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 616, 0, 5, 0, 0, 0, 0, 0, 3368, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8116, 8190, 0, 8190, 8192, 8192, 8192] dup_rows_per_dst=[10, 2, 286, 0, 0, 16, 1242, 6716] combine_rows_per_src=[8192, 8116, 8190, 0, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=227475456 dispatch_meta_bytes=229224 combine_payload_bytes=227475456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=227704680 ingress_bytes=252641280 link_bytes=252641280 token_bytes=455180136 weight_movement_bytes=25165824 total_nvlink_bytes=480345960 weight_to_token_ratio=0.055288 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.922308230e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56722 payload_rows_remote=48530 dup_rows=8814 dup_rows_remote=6760 dedup_ratio=0.134491 router_tokens_per_expert=[0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 4081, 0, 0, 4083, 0, 0, 4089, 999, 0, 32, 4082, 0, 3, 0, 4087, 0, 0, 2719, 0, 0, 3, 0, 4092, 0, 0, 17, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3036, 0, 1, 0, 0, 0, 0, 97, 1331, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8174, 8162, 8192, 5878, 8192, 8172, 7288, 2664] dup_rows_per_dst=[0, 14, 2054, 2042, 26, 0, 4678, 0] combine_rows_per_src=[8174, 8162, 8192, 5878, 8192, 8172, 7288, 2664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57482 payload_rows_remote=49290 dup_rows=8054 dup_rows_remote=7900 dedup_ratio=0.122894 router_tokens_per_expert=[0, 95, 0, 0, 9, 0, 0, 0, 0, 4064, 2, 0, 0, 0, 0, 0, 0, 2, 3, 0, 7, 4069, 3955, 3, 4089, 248, 3019, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 0, 0, 0, 1449, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8154, 7152, 8192, 7192, 2898, 7602, 8100] dup_rows_per_dst=[154, 882, 30, 6970, 0, 0, 0, 18] combine_rows_per_src=[8192, 8154, 7152, 8192, 7192, 2898, 7602, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=201719808 dispatch_meta_bytes=232016 combine_payload_bytes=201719808 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=201951824 ingress_bytes=252051456 link_bytes=252051456 token_bytes=403671632 weight_movement_bytes=50331648 total_nvlink_bytes=454003280 weight_to_token_ratio=0.124685 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.917820351e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=201080832 dispatch_meta_bytes=226224 combine_payload_bytes=201080832 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201307056 ingress_bytes=226246656 link_bytes=226246656 token_bytes=402387888 weight_movement_bytes=25165824 total_nvlink_bytes=427553712 weight_to_token_ratio=0.062541 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=25984 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.721475639e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=200998912 dispatch_meta_bytes=224408 combine_payload_bytes=200998912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201223320 ingress_bytes=226164736 link_bytes=226164736 token_bytes=402222232 weight_movement_bytes=25165824 total_nvlink_bytes=427388056 weight_to_token_ratio=0.062567 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=23296 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.720852322e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=198778880 dispatch_meta_bytes=221160 combine_payload_bytes=198778880 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=199000040 ingress_bytes=299442176 link_bytes=299442176 token_bytes=397778920 weight_movement_bytes=100663296 total_nvlink_bytes=498442216 weight_to_token_ratio=0.253063 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.278408973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59638 payload_rows_remote=56408 dup_rows=5898 dup_rows_remote=5898 dedup_ratio=0.089996 router_tokens_per_expert=[0, 21, 0, 0, 0, 121, 1, 0, 0, 4055, 2, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4086, 4059, 0, 4023, 30, 1585, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 0, 2716, 0, 0, 0, 130, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 28] payload_rows_per_dst=[8174, 8112, 8124, 3230, 7550, 8146, 8192, 8110] dup_rows_per_dst=[284, 2, 2, 0, 0, 5332, 260, 18] combine_rows_per_src=[8174, 8112, 8124, 3230, 7550, 8146, 8192, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=201891840 dispatch_meta_bytes=228760 combine_payload_bytes=201891840 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202120600 ingress_bytes=227057664 link_bytes=227057664 token_bytes=404012440 weight_movement_bytes=25165824 total_nvlink_bytes=429178264 weight_to_token_ratio=0.062290 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.727646472e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64054 payload_rows_remote=55898 dup_rows=1482 dup_rows_remote=1482 dedup_ratio=0.022614 router_tokens_per_expert=[0, 0, 0, 0, 0, 22, 0, 4, 0, 4040, 0, 0, 0, 0, 0, 0, 4, 0, 4, 77, 0, 4013, 4063, 0, 3543, 0, 4074, 0, 0, 0, 3961, 0, 6, 0, 0, 3, 0, 0, 0, 4069, 0, 12, 5, 4, 0, 0, 6, 0, 0, 0, 133, 6, 4, 0, 4083, 0, 2, 0, 33, 25, 8, 1, 0, 563] payload_rows_per_dst=[8050, 8080, 8142, 8148, 8156, 7110, 8190, 8178] dup_rows_per_dst=[28, 0, 154, 0, 0, 30, 262, 1008] combine_rows_per_src=[8050, 8080, 8142, 8148, 8156, 7110, 8190, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63992 payload_rows_remote=55802 dup_rows=1544 dup_rows_remote=72 dedup_ratio=0.023560 router_tokens_per_expert=[0, 16, 0, 0, 0, 0, 0, 0, 0, 3910, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3913, 0, 4093, 45, 3858, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 738, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7844, 7826, 7772, 7990, 8190, 8192, 7986] dup_rows_per_dst=[32, 2, 0, 34, 0, 1472, 2, 2] combine_rows_per_src=[8192, 7844, 7826, 7772, 7990, 8190, 8192, 7986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57848 payload_rows_remote=49656 dup_rows=7688 dup_rows_remote=7656 dedup_ratio=0.117310 router_tokens_per_expert=[0, 20, 0, 0, 0, 0, 0, 0, 0, 4095, 17, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4066, 4093, 0, 4071, 17, 4032, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 129] payload_rows_per_dst=[8132, 8192, 8188, 8190, 8192, 630, 8192, 8132] dup_rows_per_dst=[40, 32, 0, 7440, 0, 2, 0, 174] combine_rows_per_src=[8132, 8192, 8188, 8190, 8192, 630, 8192, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231047168 dispatch_meta_bytes=249224 combine_payload_bytes=231047168 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=231296392 ingress_bytes=356876288 link_bytes=356876288 token_bytes=462343560 weight_movement_bytes=125829120 total_nvlink_bytes=588172680 weight_to_token_ratio=0.272155 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24064 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.715416204e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=228958208 dispatch_meta_bytes=229520 combine_payload_bytes=228958208 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229187728 ingress_bytes=254124032 link_bytes=254124032 token_bytes=458145936 weight_movement_bytes=25165824 total_nvlink_bytes=483311760 weight_to_token_ratio=0.054930 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.933590259e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55222 payload_rows_remote=47470 dup_rows=10314 dup_rows_remote=10164 dedup_ratio=0.157379 router_tokens_per_expert=[0, 342, 0, 11, 0, 16, 5, 0, 45, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 4031, 0, 3896, 162, 2255, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 2435, 0, 0, 0, 0, 3489, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4083, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[6760, 8068, 8066, 324, 7894, 8192, 8166, 7752] dup_rows_per_dst=[738, 46, 1422, 0, 1336, 6578, 44, 150] combine_rows_per_src=[6760, 8068, 8066, 324, 7894, 8192, 8166, 7752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=228564992 dispatch_meta_bytes=223496 combine_payload_bytes=228564992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=228788488 ingress_bytes=253730816 link_bytes=253730816 token_bytes=457353480 weight_movement_bytes=25165824 total_nvlink_bytes=482519304 weight_to_token_ratio=0.055025 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.930598340e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64330 payload_rows_remote=56138 dup_rows=1206 dup_rows_remote=1206 dedup_ratio=0.018402 router_tokens_per_expert=[0, 6, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4096, 4094, 0, 3777, 14, 4083, 0, 0, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7226, 8080, 8104, 8192, 8152] dup_rows_per_dst=[14, 0, 108, 6, 906, 172, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7226, 8080, 8104, 8192, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64302 payload_rows_remote=56172 dup_rows=1234 dup_rows_remote=1234 dedup_ratio=0.018829 router_tokens_per_expert=[0, 118, 10, 2, 5, 65, 0, 0, 0, 3935, 23, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4077, 4065, 0, 3968, 185, 3817, 0, 0, 0, 3869, 0, 0, 0, 24, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 369, 2, 0, 0, 0, 0, 36, 0, 0, 0, 4085, 0, 0, 0, 0, 9, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7914, 8130, 7878, 8192, 8050, 8190, 7756] dup_rows_per_dst=[362, 16, 0, 126, 48, 630, 52, 0] combine_rows_per_src=[8192, 7914, 8130, 7878, 8192, 8050, 8190, 7756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=203390976 dispatch_meta_bytes=229248 combine_payload_bytes=203390976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203620224 ingress_bytes=228556800 link_bytes=228556800 token_bytes=407011200 weight_movement_bytes=25165824 total_nvlink_bytes=432177024 weight_to_token_ratio=0.061831 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.739053165e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62642 payload_rows_remote=54450 dup_rows=2894 dup_rows_remote=848 dedup_ratio=0.044159 router_tokens_per_expert=[28, 0, 0, 0, 4096, 7, 4095, 3961, 1, 0, 4087, 0, 4002, 0, 4, 0, 0, 1, 0, 3178, 973, 3959, 0, 0, 0, 3, 198, 0, 0, 0, 1, 32, 1, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8142, 8182, 6364, 7518, 8134, 7918, 8192] dup_rows_per_dst=[2046, 662, 122, 58, 0, 0, 2, 4] combine_rows_per_src=[8192, 8142, 8182, 6364, 7518, 8134, 7918, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=194437120 dispatch_meta_bytes=230536 combine_payload_bytes=194437120 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=194667656 ingress_bytes=244768768 link_bytes=244768768 token_bytes=389104776 weight_movement_bytes=50331648 total_nvlink_bytes=439436424 weight_to_token_ratio=0.129352 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.862407510e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229941248 dispatch_meta_bytes=229376 combine_payload_bytes=229941248 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=230170624 ingress_bytes=280272896 link_bytes=280272896 token_bytes=460111872 weight_movement_bytes=50331648 total_nvlink_bytes=510443520 weight_to_token_ratio=0.109390 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.132552901e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65054 payload_rows_remote=56862 dup_rows=482 dup_rows_remote=34 dedup_ratio=0.007355 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 4096, 0, 4003, 0, 1, 0, 0, 4, 0, 2761, 1218, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8008, 7958, 8192, 8192, 8192, 8130, 8190] dup_rows_per_dst=[0, 0, 8, 448, 26, 0, 0, 0] combine_rows_per_src=[8192, 8008, 7958, 8192, 8192, 8192, 8130, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230080512 dispatch_meta_bytes=229624 combine_payload_bytes=230080512 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=230310136 ingress_bytes=305577984 link_bytes=305577984 token_bytes=460390648 weight_movement_bytes=75497472 total_nvlink_bytes=535888120 weight_to_token_ratio=0.163986 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.325095382e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=223027200 dispatch_meta_bytes=221192 combine_payload_bytes=223027200 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=223248392 ingress_bytes=348856320 link_bytes=348856320 token_bytes=446275592 weight_movement_bytes=125829120 total_nvlink_bytes=572104712 weight_to_token_ratio=0.281954 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.654393514e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64950 payload_rows_remote=56780 dup_rows=586 dup_rows_remote=566 dedup_ratio=0.008942 router_tokens_per_expert=[0, 0, 0, 4, 4078, 8, 4087, 3968, 0, 0, 4081, 1, 3998, 9, 0, 0, 0, 0, 13, 2677, 1511, 4028, 0, 0, 0, 0, 105, 0, 0, 7, 0, 21, 0, 9, 0, 2, 0, 0, 0, 3, 0, 0, 0, 1, 0, 4075, 5, 27, 9, 6, 16, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11] payload_rows_per_dst=[7960, 8014, 8168, 8192, 8170, 8180, 8090, 8176] dup_rows_per_dst=[0, 2, 234, 230, 20, 36, 44, 20] combine_rows_per_src=[7960, 8014, 8168, 8192, 8170, 8180, 8090, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65236 payload_rows_remote=57048 dup_rows=300 dup_rows_remote=298 dedup_ratio=0.004578 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4094, 3991, 0, 0, 4096, 0, 4092, 0, 1, 0, 0, 0, 0, 974, 3249, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7982, 8184, 8192, 8192, 8192, 8188, 8118, 8188] dup_rows_per_dst=[0, 2, 254, 0, 16, 2, 26, 0] combine_rows_per_src=[7982, 8184, 8192, 8192, 8192, 8188, 8118, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64778 payload_rows_remote=56992 dup_rows=758 dup_rows_remote=754 dedup_ratio=0.011566 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 4096, 3928, 0, 0, 4096, 0, 3879, 0, 16, 0, 0, 0, 0, 2287, 2006, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7856, 7786, 8184, 8190, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 4, 402, 8, 344, 0, 0, 0] combine_rows_per_src=[7856, 7786, 8184, 8190, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232906752 dispatch_meta_bytes=227584 combine_payload_bytes=232906752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233134336 ingress_bytes=258072576 link_bytes=258072576 token_bytes=466041088 weight_movement_bytes=25165824 total_nvlink_bytes=491206912 weight_to_token_ratio=0.053999 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.963634117e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232570880 dispatch_meta_bytes=229384 combine_payload_bytes=232570880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=232800264 ingress_bytes=257736704 link_bytes=257736704 token_bytes=465371144 weight_movement_bytes=25165824 total_nvlink_bytes=490536968 weight_to_token_ratio=0.054077 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.961078519e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63658 payload_rows_remote=55468 dup_rows=1878 dup_rows_remote=1878 dedup_ratio=0.028656 router_tokens_per_expert=[7, 0, 0, 0, 4094, 0, 4095, 3997, 0, 0, 4096, 0, 4008, 0, 1, 0, 0, 0, 0, 4076, 0, 3857, 0, 0, 0, 0, 1, 0, 0, 0, 159, 0, 126, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 128, 22, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7010, 8018, 7890, 8192, 8192, 8176, 7990, 8190] dup_rows_per_dst=[0, 436, 0, 316, 278, 254, 594, 0] combine_rows_per_src=[7010, 8018, 7890, 8192, 8192, 8176, 7990, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233668608 dispatch_meta_bytes=229384 combine_payload_bytes=233668608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233897992 ingress_bytes=258834432 link_bytes=258834432 token_bytes=467566600 weight_movement_bytes=25165824 total_nvlink_bytes=492732424 weight_to_token_ratio=0.053823 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.969430961e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233439232 dispatch_meta_bytes=230984 combine_payload_bytes=233439232 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=233670216 ingress_bytes=283770880 link_bytes=283770880 token_bytes=467109448 weight_movement_bytes=50331648 total_nvlink_bytes=517441096 weight_to_token_ratio=0.107751 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.159168518e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65304 payload_rows_remote=57138 dup_rows=232 dup_rows_remote=152 dedup_ratio=0.003540 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 4096, 0, 3991, 0, 15, 0, 0, 0, 0, 3151, 942, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8012, 8186, 8192, 8192, 8172, 8166, 8192] dup_rows_per_dst=[0, 0, 0, 140, 12, 0, 80, 0] combine_rows_per_src=[8192, 8012, 8186, 8192, 8192, 8172, 8166, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64816 payload_rows_remote=57070 dup_rows=720 dup_rows_remote=710 dedup_ratio=0.010986 router_tokens_per_expert=[0, 4, 0, 0, 4091, 0, 4090, 4049, 0, 0, 4096, 0, 4054, 0, 79, 0, 0, 81, 0, 620, 3177, 4086, 0, 0, 0, 1, 2, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 0, 4, 0, 2, 0, 0] payload_rows_per_dst=[8104, 8192, 7746, 8192, 8192, 8010, 8190, 8190] dup_rows_per_dst=[2, 74, 10, 630, 0, 0, 2, 2] combine_rows_per_src=[8104, 8192, 7746, 8192, 8192, 8010, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53114 payload_rows_remote=44938 dup_rows=12422 dup_rows_remote=5464 dedup_ratio=0.189545 router_tokens_per_expert=[15, 14, 0, 3468, 0, 0, 10, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 2701, 0, 4095, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4083, 0, 335, 4088, 0, 0, 0, 0, 0, 0, 0, 3, 4060, 198, 2, 0, 0, 1484, 0] payload_rows_per_dst=[8176, 8190, 8190, 8150, 8192, 670, 8176, 3370] dup_rows_per_dst=[6958, 56, 5402, 0, 2, 0, 0, 4] combine_rows_per_src=[8176, 8190, 8190, 8150, 8192, 670, 8176, 3370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=227196928 dispatch_meta_bytes=229384 combine_payload_bytes=227196928 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=227426312 ingress_bytes=252362752 link_bytes=252362752 token_bytes=454623240 weight_movement_bytes=25165824 total_nvlink_bytes=479789064 weight_to_token_ratio=0.055355 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.920188954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62122 payload_rows_remote=53982 dup_rows=3414 dup_rows_remote=3414 dedup_ratio=0.052094 router_tokens_per_expert=[0, 0, 0, 2724, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1440, 0, 4096, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 4094, 0, 3789, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 0, 0, 233, 0] payload_rows_per_dst=[5448, 8190, 8192, 8140, 8190, 7578, 8192, 8192] dup_rows_per_dst=[0, 0, 2880, 0, 0, 6, 0, 528] combine_rows_per_src=[5448, 8190, 8192, 8140, 8190, 7578, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60178 payload_rows_remote=52060 dup_rows=5358 dup_rows_remote=5356 dedup_ratio=0.081757 router_tokens_per_expert=[0, 66, 0, 3245, 0, 19, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 1273, 13, 4015, 0, 0, 59, 0, 8, 4085, 0, 5, 0, 0, 0, 0, 2, 0, 2, 1, 1, 3, 4051, 0, 13, 0, 0, 0, 0, 4008, 25, 2615, 4082, 0, 0, 0, 0, 0, 0, 4, 0, 4080, 15, 0, 1, 0, 1066, 0] payload_rows_per_dst=[6598, 8036, 7626, 8186, 8118, 5266, 8172, 8176] dup_rows_per_dst=[62, 2, 2076, 1028, 2, 40, 0, 2148] combine_rows_per_src=[6598, 8036, 7626, 8186, 8118, 5266, 8172, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234037248 dispatch_meta_bytes=229160 combine_payload_bytes=234037248 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=234266408 ingress_bytes=284368896 link_bytes=284368896 token_bytes=468303656 weight_movement_bytes=50331648 total_nvlink_bytes=518635304 weight_to_token_ratio=0.107477 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.163718728e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233758720 dispatch_meta_bytes=231120 combine_payload_bytes=233758720 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=233989840 ingress_bytes=309256192 link_bytes=309256192 token_bytes=467748560 weight_movement_bytes=75497472 total_nvlink_bytes=543246032 weight_to_token_ratio=0.161406 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.353082295e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=184066048 dispatch_meta_bytes=201608 combine_payload_bytes=184066048 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=184267656 ingress_bytes=209231872 link_bytes=209231872 token_bytes=368333704 weight_movement_bytes=25165824 total_nvlink_bytes=393499528 weight_to_token_ratio=0.068323 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=26496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592012792e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57146 payload_rows_remote=49038 dup_rows=8390 dup_rows_remote=1538 dedup_ratio=0.128021 router_tokens_per_expert=[5, 0, 0, 4014, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 88, 0, 4096, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 3869, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 2837, 448, 0, 0, 0, 1048, 0] payload_rows_per_dst=[8028, 8192, 176, 8192, 8174, 8108, 8192, 8084] dup_rows_per_dst=[10, 74, 0, 860, 0, 6852, 12, 582] combine_rows_per_src=[8028, 8192, 176, 8192, 8174, 8108, 8192, 8084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55280 payload_rows_remote=47088 dup_rows=10256 dup_rows_remote=10144 dedup_ratio=0.156494 router_tokens_per_expert=[0, 0, 0, 3778, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 1848, 0, 4096, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 708, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 786, 0, 0, 0, 1121, 0] payload_rows_per_dst=[7804, 8192, 8192, 7998, 8192, 1416, 8182, 5304] dup_rows_per_dst=[3904, 112, 4440, 0, 206, 0, 102, 1492] combine_rows_per_src=[7804, 8192, 8192, 7998, 8192, 1416, 8182, 5304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=221110272 dispatch_meta_bytes=229584 combine_payload_bytes=221110272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=221339856 ingress_bytes=246276096 link_bytes=246276096 token_bytes=442450128 weight_movement_bytes=25165824 total_nvlink_bytes=467615952 weight_to_token_ratio=0.056878 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.873876534e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=213237760 dispatch_meta_bytes=229664 combine_payload_bytes=213237760 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213467424 ingress_bytes=238403584 link_bytes=238403584 token_bytes=426705184 weight_movement_bytes=25165824 total_nvlink_bytes=451871008 weight_to_token_ratio=0.058977 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.813975814e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61516 payload_rows_remote=53328 dup_rows=4020 dup_rows_remote=3496 dedup_ratio=0.061340 router_tokens_per_expert=[8, 0, 0, 3987, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 4096, 0, 0, 0, 0, 0, 3306, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 1471, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 261, 0, 3, 0, 0, 0] payload_rows_per_dst=[7990, 7724, 6412, 6638, 8180, 8192, 8192, 8188] dup_rows_per_dst=[0, 84, 0, 0, 0, 3410, 2, 524] combine_rows_per_src=[7990, 7724, 6412, 6638, 8180, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=200859648 dispatch_meta_bytes=202304 combine_payload_bytes=200859648 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=201061952 ingress_bytes=251191296 link_bytes=251191296 token_bytes=401921600 weight_movement_bytes=50331648 total_nvlink_bytes=452253248 weight_to_token_ratio=0.125228 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.911275527e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=192872448 dispatch_meta_bytes=228928 combine_payload_bytes=192872448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193101376 ingress_bytes=218038272 link_bytes=218038272 token_bytes=385973824 weight_movement_bytes=25165824 total_nvlink_bytes=411139648 weight_to_token_ratio=0.065201 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.659019321e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57652 payload_rows_remote=49546 dup_rows=7884 dup_rows_remote=7884 dedup_ratio=0.120300 router_tokens_per_expert=[0, 0, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 4095, 0, 3908, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 56, 0, 0, 0, 5, 0] payload_rows_per_dst=[7612, 8190, 1108, 8180, 8102, 8192, 8106, 8162] dup_rows_per_dst=[0, 0, 0, 2, 0, 7816, 0, 66] combine_rows_per_src=[7612, 8190, 1108, 8180, 8102, 8192, 8106, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59998 payload_rows_remote=51812 dup_rows=5538 dup_rows_remote=2882 dedup_ratio=0.084503 router_tokens_per_expert=[1, 0, 0, 1472, 0, 0, 0, 0, 0, 150, 0, 0, 0, 1, 0, 0, 0, 1329, 0, 4092, 0, 0, 0, 0, 1, 4075, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 6, 0, 28, 4096, 0, 3961, 4090, 0, 0, 0, 0, 14, 0, 0, 0, 3828, 1307, 0, 0, 0, 223, 0] payload_rows_per_dst=[2946, 8192, 8186, 8160, 8178, 7978, 8188, 8170] dup_rows_per_dst=[0, 302, 2656, 2, 0, 12, 20, 2546] combine_rows_per_src=[2946, 8192, 8186, 8160, 8178, 7978, 8188, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63608 payload_rows_remote=55416 dup_rows=1928 dup_rows_remote=318 dedup_ratio=0.029419 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 11, 4081, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 4088, 11, 0, 34, 4, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 92, 0, 0, 3959, 3, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8176, 8188, 7994, 7700, 6976] dup_rows_per_dst=[1610, 0, 0, 254, 36, 12, 0, 16] combine_rows_per_src=[8192, 8190, 8192, 8176, 8188, 7994, 7700, 6976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=218431488 dispatch_meta_bytes=227296 combine_payload_bytes=218431488 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=218658784 ingress_bytes=319094784 link_bytes=319094784 token_bytes=437090272 weight_movement_bytes=100663296 total_nvlink_bytes=537753568 weight_to_token_ratio=0.230303 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=24576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.427942613e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64384 payload_rows_remote=56196 dup_rows=1152 dup_rows_remote=1152 dedup_ratio=0.017578 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3526, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 571, 0, 0, 4096, 0, 0, 0, 1] payload_rows_per_dst=[7052, 8192, 8192, 8188, 8192, 8184, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 8, 1144] combine_rows_per_src=[7052, 8192, 8192, 8188, 8192, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64810 payload_rows_remote=56618 dup_rows=726 dup_rows_remote=638 dedup_ratio=0.011078 router_tokens_per_expert=[3, 3, 0, 3, 0, 17, 4086, 3993, 0, 0, 0, 0, 174, 12, 9, 0, 0, 0, 0, 3, 0, 6, 0, 48, 4, 6, 4050, 0, 0, 3893, 16, 8, 13, 4061, 0, 34, 0, 4082, 10, 1, 4081, 21, 0, 0, 0, 0, 39, 0, 0, 0, 0, 1, 0, 0, 0, 34, 46, 0, 0, 4011, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8168, 7820, 8192, 8162, 8036, 8048] dup_rows_per_dst=[32, 298, 68, 34, 88, 120, 20, 66] combine_rows_per_src=[8192, 8192, 8168, 7820, 8192, 8162, 8036, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=202940416 dispatch_meta_bytes=229720 combine_payload_bytes=202940416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203170136 ingress_bytes=228106240 link_bytes=228106240 token_bytes=406110552 weight_movement_bytes=25165824 total_nvlink_bytes=431276376 weight_to_token_ratio=0.061968 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.735624924e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=226983936 dispatch_meta_bytes=222936 combine_payload_bytes=226983936 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=227206872 ingress_bytes=277315584 link_bytes=277315584 token_bytes=454190808 weight_movement_bytes=50331648 total_nvlink_bytes=504522456 weight_to_token_ratio=0.110816 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.110051174e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=212221952 dispatch_meta_bytes=218776 combine_payload_bytes=212221952 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=212440728 ingress_bytes=287719424 link_bytes=287719424 token_bytes=424662680 weight_movement_bytes=75497472 total_nvlink_bytes=500160152 weight_to_token_ratio=0.177782 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.189212375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65074 payload_rows_remote=56882 dup_rows=462 dup_rows_remote=462 dedup_ratio=0.007050 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3924, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 165, 0, 28, 0, 0, 0, 0, 0, 32, 0, 0, 4044, 0, 0, 0, 0] payload_rows_per_dst=[7848, 8192, 8192, 8190, 8178, 8192, 8192, 8090] dup_rows_per_dst=[0, 0, 0, 0, 0, 14, 386, 62] combine_rows_per_src=[7848, 8192, 8192, 8190, 8178, 8192, 8192, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65442 payload_rows_remote=57250 dup_rows=94 dup_rows_remote=90 dedup_ratio=0.001434 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 4079, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8186, 8192, 8118, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 0, 90] combine_rows_per_src=[8178, 8186, 8192, 8118, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=230178816 dispatch_meta_bytes=229392 combine_payload_bytes=230178816 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=230408208 ingress_bytes=280510464 link_bytes=280510464 token_bytes=460587024 weight_movement_bytes=50331648 total_nvlink_bytes=510918672 weight_to_token_ratio=0.109277 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.134360519e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=231907328 dispatch_meta_bytes=229024 combine_payload_bytes=231907328 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=232136352 ingress_bytes=282238976 link_bytes=282238976 token_bytes=464043680 weight_movement_bytes=50331648 total_nvlink_bytes=514375328 weight_to_token_ratio=0.108463 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.147512498e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64786 payload_rows_remote=56596 dup_rows=750 dup_rows_remote=684 dedup_ratio=0.011444 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4033, 0, 0, 0, 4096, 0, 0, 0, 4021, 0, 0, 4096, 10, 0, 0, 0, 0, 26, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8108, 7906, 8144, 8078, 8042, 8126, 8192, 8190] dup_rows_per_dst=[0, 0, 42, 288, 48, 72, 234, 66] combine_rows_per_src=[8108, 7906, 8144, 8078, 8042, 8126, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=232988672 dispatch_meta_bytes=229376 combine_payload_bytes=232988672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233218048 ingress_bytes=258154496 link_bytes=258154496 token_bytes=466206720 weight_movement_bytes=25165824 total_nvlink_bytes=491372544 weight_to_token_ratio=0.053980 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.964257434e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229360 combine_payload_bytes=234496000 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234725360 ingress_bytes=309993472 link_bytes=309993472 token_bytes=469221360 weight_movement_bytes=75497472 total_nvlink_bytes=544718832 weight_to_token_ratio=0.160899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.358692144e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65470 payload_rows_remote=57278 dup_rows=66 dup_rows_remote=66 dedup_ratio=0.001007 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4082, 0, 0, 0, 4094, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8156, 8192, 8166, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 62, 0, 0] combine_rows_per_src=[8156, 8192, 8166, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60552 payload_rows_remote=52360 dup_rows=4984 dup_rows_remote=4980 dedup_ratio=0.076050 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 0, 139, 4093, 0, 0, 0, 3726, 0, 0, 15, 0, 0, 4084, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 3962, 4092, 0, 0, 0, 0, 4065, 0, 0, 17, 0, 0, 81, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 3994, 0, 0] payload_rows_per_dst=[8192, 8188, 8160, 8166, 8112, 7282, 4464, 7988] dup_rows_per_dst=[4, 4134, 4, 804, 0, 34, 4, 0] combine_rows_per_src=[8192, 8188, 8160, 8166, 8112, 7282, 4464, 7988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=231817216 dispatch_meta_bytes=229120 combine_payload_bytes=231817216 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=232046336 ingress_bytes=282148864 link_bytes=282148864 token_bytes=463863552 weight_movement_bytes=50331648 total_nvlink_bytes=514195200 weight_to_token_ratio=0.108505 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.146826850e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65496 payload_rows_remote=57316 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000610 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4084, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 1, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4083, 0, 7, 0, 0] payload_rows_per_dst=[8168, 8188, 8180, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 18, 0, 20] combine_rows_per_src=[8168, 8188, 8180, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64680 payload_rows_remote=56490 dup_rows=856 dup_rows_remote=856 dedup_ratio=0.013062 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 3, 0, 3698, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 0, 0, 0, 0, 0, 4060, 0, 0] payload_rows_per_dst=[8192, 7414, 8192, 8190, 8192, 8188, 8192, 8120] dup_rows_per_dst=[0, 12, 0, 0, 0, 4, 840, 0] combine_rows_per_src=[8192, 7414, 8192, 8190, 8192, 8188, 8192, 8120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64952 payload_rows_remote=56766 dup_rows=584 dup_rows_remote=576 dedup_ratio=0.008911 router_tokens_per_expert=[0, 1, 10, 23, 15, 0, 0, 7, 0, 4068, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 4086, 0, 7, 0, 0, 0, 0, 0, 0, 0, 10, 3, 3873, 4081, 0, 0, 10, 0, 4086, 0, 1, 0, 0, 11, 0, 0, 4079, 0, 0, 0, 12, 4, 0, 0, 0, 228, 0, 9, 0, 0, 6, 0, 4079, 3, 5] payload_rows_per_dst=[8192, 8102, 8182, 7766, 8186, 8180, 8168, 8176] dup_rows_per_dst=[82, 0, 4, 6, 8, 0, 456, 28] combine_rows_per_src=[8192, 8102, 8182, 7766, 8186, 8180, 8168, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234610688 dispatch_meta_bytes=229376 combine_payload_bytes=234610688 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234840064 ingress_bytes=259776512 link_bytes=259776512 token_bytes=469450752 weight_movement_bytes=25165824 total_nvlink_bytes=494616576 weight_to_token_ratio=0.053607 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.976599101e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=214466560 dispatch_meta_bytes=229360 combine_payload_bytes=214466560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=214695920 ingress_bytes=239632384 link_bytes=239632384 token_bytes=429162480 weight_movement_bytes=25165824 total_nvlink_bytes=454328304 weight_to_token_ratio=0.058639 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.823325562e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234766336 dispatch_meta_bytes=229424 combine_payload_bytes=234766336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234995760 ingress_bytes=259932160 link_bytes=259932160 token_bytes=469762096 weight_movement_bytes=25165824 total_nvlink_bytes=494927920 weight_to_token_ratio=0.053571 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.977783403e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65326 payload_rows_remote=57332 dup_rows=210 dup_rows_remote=210 dedup_ratio=0.003204 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 100, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 7994, 8192, 8190, 8182, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 200, 10, 0] combine_rows_per_src=[8192, 7994, 8192, 8190, 8182, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=231383040 dispatch_meta_bytes=229384 combine_payload_bytes=231383040 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=231612424 ingress_bytes=256548864 link_bytes=256548864 token_bytes=462995464 weight_movement_bytes=25165824 total_nvlink_bytes=488161288 weight_to_token_ratio=0.054354 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.952040429e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64698 payload_rows_remote=56506 dup_rows=838 dup_rows_remote=828 dedup_ratio=0.012787 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 245, 4096, 0, 0, 0, 3957, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 3756, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 8042, 8192, 8192, 8192, 7512] dup_rows_per_dst=[0, 220, 0, 0, 0, 10, 608, 0] combine_rows_per_src=[8192, 8184, 8192, 8042, 8192, 8192, 8192, 7512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232513536 dispatch_meta_bytes=229368 combine_payload_bytes=232513536 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=232742904 ingress_bytes=282845184 link_bytes=282845184 token_bytes=465256440 weight_movement_bytes=50331648 total_nvlink_bytes=515588088 weight_to_token_ratio=0.108180 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.152125041e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60720 payload_rows_remote=53864 dup_rows=4816 dup_rows_remote=2104 dedup_ratio=0.073486 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 51, 4096, 0, 1, 0, 4020, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 4096, 0, 0, 0, 6, 4085, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 0, 0, 3428, 0, 0] payload_rows_per_dst=[8104, 4912, 8192, 8090, 8182, 8192, 8192, 6856] dup_rows_per_dst=[2, 104, 0, 0, 88, 422, 1488, 2712] combine_rows_per_src=[8104, 4912, 8192, 8090, 8182, 8192, 8192, 6856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234831872 dispatch_meta_bytes=230168 combine_payload_bytes=234831872 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=235062040 ingress_bytes=335495168 link_bytes=335495168 token_bytes=469893912 weight_movement_bytes=100663296 total_nvlink_bytes=570557208 weight_to_token_ratio=0.214226 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.552730585e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=231448576 dispatch_meta_bytes=229336 combine_payload_bytes=231448576 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=231677912 ingress_bytes=281780224 link_bytes=281780224 token_bytes=463126488 weight_movement_bytes=50331648 total_nvlink_bytes=513458136 weight_to_token_ratio=0.108678 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.144021926e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=220626944 dispatch_meta_bytes=223872 combine_payload_bytes=220626944 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=220850816 ingress_bytes=245792768 link_bytes=245792768 token_bytes=441477760 weight_movement_bytes=25165824 total_nvlink_bytes=466643584 weight_to_token_ratio=0.057004 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.870198967e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65458 payload_rows_remote=57266 dup_rows=78 dup_rows_remote=78 dedup_ratio=0.001190 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 13, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4095, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8126, 8186, 8192, 8192, 8188] dup_rows_per_dst=[0, 76, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8126, 8186, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53026 payload_rows_remote=44834 dup_rows=12510 dup_rows_remote=12490 dedup_ratio=0.190887 router_tokens_per_expert=[7, 4096, 0, 0, 0, 3, 0, 0, 4081, 0, 0, 675, 0, 0, 2315, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 537, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 569, 4086, 0] payload_rows_per_dst=[8192, 8192, 8192, 8172, 8192, 1074, 8192, 2820] dup_rows_per_dst=[20, 11366, 742, 48, 30, 0, 0, 304] combine_rows_per_src=[8192, 8192, 8192, 8172, 8192, 1074, 8192, 2820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64880 payload_rows_remote=56688 dup_rows=656 dup_rows_remote=622 dedup_ratio=0.010010 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 3998, 3, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4092, 3, 0, 0, 10, 4091, 0, 0, 0, 0, 0, 143, 0, 4080, 0, 0, 0, 0, 0, 0, 1, 0, 173, 0, 0, 0, 0, 0, 0, 3882, 0, 0] payload_rows_per_dst=[8190, 8002, 8192, 8156, 8192, 8192, 8192, 7764] dup_rows_per_dst=[4, 2, 34, 0, 16, 254, 346, 0] combine_rows_per_src=[8190, 8002, 8192, 8156, 8192, 8192, 8192, 7764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58538 payload_rows_remote=50346 dup_rows=6998 dup_rows_remote=6998 dedup_ratio=0.106781 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1663, 0, 0, 2168, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 19, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 35, 4096, 0] payload_rows_per_dst=[8192, 8102, 8088, 8192, 8192, 1388, 8192, 8192] dup_rows_per_dst=[0, 6840, 0, 0, 0, 88, 0, 70] combine_rows_per_src=[8192, 8102, 8088, 8192, 8192, 1388, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57328 payload_rows_remote=49136 dup_rows=8208 dup_rows_remote=8162 dedup_ratio=0.125244 router_tokens_per_expert=[0, 4076, 8, 0, 0, 0, 1, 13, 4082, 0, 8, 86, 0, 0, 0, 1, 0, 0, 0, 4066, 0, 7, 1, 0, 3, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 26, 0, 0, 0, 3679, 0, 8, 4079, 0, 0, 22, 13, 0, 4047, 76, 0, 24, 0, 0, 3971, 0, 1, 0, 368, 4082, 0] payload_rows_per_dst=[8188, 190, 8148, 8180, 8192, 8180, 8192, 8058] dup_rows_per_dst=[8, 0, 0, 2, 46, 7358, 172, 622] combine_rows_per_src=[8188, 190, 8148, 8180, 8192, 8180, 8192, 8058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234561536 dispatch_meta_bytes=229376 combine_payload_bytes=234561536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234790912 ingress_bytes=259727360 link_bytes=259727360 token_bytes=469352448 weight_movement_bytes=25165824 total_nvlink_bytes=494518272 weight_to_token_ratio=0.053618 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.976225111e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=183640064 dispatch_meta_bytes=229296 combine_payload_bytes=183640064 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=183869360 ingress_bytes=208805888 link_bytes=208805888 token_bytes=367509424 weight_movement_bytes=25165824 total_nvlink_bytes=392675248 weight_to_token_ratio=0.068477 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588771546e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232194048 dispatch_meta_bytes=229240 combine_payload_bytes=232194048 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=232423288 ingress_bytes=282525696 link_bytes=282525696 token_bytes=464617336 weight_movement_bytes=50331648 total_nvlink_bytes=514948984 weight_to_token_ratio=0.108329 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.149694106e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59598 payload_rows_remote=51406 dup_rows=5938 dup_rows_remote=1604 dedup_ratio=0.090607 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2145, 0, 0, 22, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 802, 4096, 0] payload_rows_per_dst=[8192, 8192, 7972, 8192, 8192, 2474, 8192, 8192] dup_rows_per_dst=[0, 4334, 0, 0, 0, 0, 0, 1604] combine_rows_per_src=[8192, 8192, 7972, 8192, 8192, 2474, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=206217216 dispatch_meta_bytes=229376 combine_payload_bytes=206217216 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=206446592 ingress_bytes=256548864 link_bytes=256548864 token_bytes=412663808 weight_movement_bytes=50331648 total_nvlink_bytes=462995456 weight_to_token_ratio=0.121968 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.952040429e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64892 payload_rows_remote=56700 dup_rows=644 dup_rows_remote=358 dedup_ratio=0.009827 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2888, 0, 0, 888, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 143, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 170, 4096, 0] payload_rows_per_dst=[8192, 7552, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 0, 0, 0, 18, 286, 0, 340] combine_rows_per_src=[8192, 7552, 8192, 8192, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56666 payload_rows_remote=48502 dup_rows=8870 dup_rows_remote=8652 dedup_ratio=0.135345 router_tokens_per_expert=[0, 4096, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 14, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 3923, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 13, 4096, 0] payload_rows_per_dst=[8146, 32, 8192, 8192, 7728, 8192, 8020, 8164] dup_rows_per_dst=[8, 0, 0, 0, 334, 8310, 0, 218] combine_rows_per_src=[8146, 32, 8192, 8192, 7728, 8192, 8020, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=201261056 dispatch_meta_bytes=229192 combine_payload_bytes=201261056 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201490248 ingress_bytes=226426880 link_bytes=226426880 token_bytes=402751304 weight_movement_bytes=25165824 total_nvlink_bytes=427917128 weight_to_token_ratio=0.062485 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.722846935e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6072703768 prefetch_bytes=352321536 reduce_bytes=452984832 weight_movement_bytes=805306368 egress_bytes=3038067480 ingress_bytes=3839942656 link_bytes=3839942656 total_nvlink_bytes=6878010136 weight_to_token_ratio=0.132611 prefetch_share_of_ingress=0.091752 dedup_ratio=0.170748 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=18 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039062 gemm_inner_pad_upper_bound=16891 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.826095425e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6959137360 prefetch_bytes=402653184 reduce_bytes=377487360 weight_movement_bytes=780140544 egress_bytes=3481403984 ingress_bytes=4257873920 link_bytes=4257873920 total_nvlink_bytes=7739277904 weight_to_token_ratio=0.112103 prefetch_share_of_ingress=0.094567 dedup_ratio=0.075226 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=15 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=15621 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.024843802e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=210558976 dispatch_meta_bytes=212040 combine_payload_bytes=210558976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=210771016 ingress_bytes=260890624 link_bytes=260890624 token_bytes=421329992 weight_movement_bytes=50331648 total_nvlink_bytes=471661640 weight_to_token_ratio=0.119459 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=16000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.985076206e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=232243200 dispatch_meta_bytes=228232 combine_payload_bytes=232243200 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=232471432 ingress_bytes=282574848 link_bytes=282574848 token_bytes=464714632 weight_movement_bytes=50331648 total_nvlink_bytes=515046280 weight_to_token_ratio=0.108307 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16896 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.150068096e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=198664192 dispatch_meta_bytes=228616 combine_payload_bytes=198664192 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=198892808 ingress_bytes=324493312 link_bytes=324493312 token_bytes=397557000 weight_movement_bytes=125829120 total_nvlink_bytes=523386120 weight_to_token_ratio=0.316506 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.469019173e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6841742240 prefetch_bytes=352321536 reduce_bytes=276824064 weight_movement_bytes=629145600 egress_bytes=3422688160 ingress_bytes=4048199680 link_bytes=4048199680 total_nvlink_bytes=7470887840 weight_to_token_ratio=0.091957 prefetch_share_of_ingress=0.087032 dedup_ratio=0.082445 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=11 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=15748 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.925132633e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59068 payload_rows_remote=50876 dup_rows=6468 dup_rows_remote=6468 dedup_ratio=0.098694 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 858, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2317, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 919, 4096, 0] payload_rows_per_dst=[8192, 1726, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 26, 4634, 0, 1808] combine_rows_per_src=[8192, 1726, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64120 payload_rows_remote=56302 dup_rows=1416 dup_rows_remote=1416 dedup_ratio=0.021606 router_tokens_per_expert=[0, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 399, 0, 0, 10, 0, 0, 0, 1, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3584, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 21, 0, 0, 270, 4087, 0] payload_rows_per_dst=[8192, 8192, 7818, 8174, 8192, 7168, 8192, 8192] dup_rows_per_dst=[14, 818, 0, 0, 10, 0, 0, 574] combine_rows_per_src=[8192, 8192, 7818, 8174, 8192, 7168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6606103176 prefetch_bytes=377487360 reduce_bytes=377487360 weight_movement_bytes=754974720 egress_bytes=3304850056 ingress_bytes=4056227840 link_bytes=4056227840 total_nvlink_bytes=7361077896 weight_to_token_ratio=0.114284 prefetch_share_of_ingress=0.093064 dedup_ratio=0.110462 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=15 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042509 gemm_inner_pad_upper_bound=15367 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.928950447e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6898840496 prefetch_bytes=352321536 reduce_bytes=402653184 weight_movement_bytes=754974720 egress_bytes=3451245488 ingress_bytes=4202569728 link_bytes=4202569728 total_nvlink_bytes=7653815216 weight_to_token_ratio=0.109435 prefetch_share_of_ingress=0.083835 dedup_ratio=0.079792 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=16 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=16510 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.998543739e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6279096432 prefetch_bytes=427819008 reduce_bytes=377487360 weight_movement_bytes=805306368 egress_bytes=3141331056 ingress_bytes=3943071744 link_bytes=3943071744 total_nvlink_bytes=7084402800 weight_to_token_ratio=0.128252 prefetch_share_of_ingress=0.108499 dedup_ratio=0.149906 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=15 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=16383 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.875138750e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=208388096 dispatch_meta_bytes=229376 combine_payload_bytes=208388096 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=208617472 ingress_bytes=258719744 link_bytes=258719744 token_bytes=417005568 weight_movement_bytes=50331648 total_nvlink_bytes=467337216 weight_to_token_ratio=0.120698 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.968558318e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=230612992 dispatch_meta_bytes=230872 combine_payload_bytes=230612992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230843864 ingress_bytes=255778816 link_bytes=255778816 token_bytes=461456856 weight_movement_bytes=25165824 total_nvlink_bytes=486622680 weight_to_token_ratio=0.054536 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.946181254e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6745901504 prefetch_bytes=377487360 reduce_bytes=276824064 weight_movement_bytes=654311424 egress_bytes=3374762432 ingress_bytes=4025450496 link_bytes=4025450496 total_nvlink_bytes=7400212928 weight_to_token_ratio=0.096994 prefetch_share_of_ingress=0.093775 dedup_ratio=0.094383 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=11 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=16256 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.914314195e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6864709848 prefetch_bytes=377487360 reduce_bytes=478150656 weight_movement_bytes=855638016 egress_bytes=3434178776 ingress_bytes=4286169088 link_bytes=4286169088 total_nvlink_bytes=7720347864 weight_to_token_ratio=0.124643 prefetch_share_of_ingress=0.088071 dedup_ratio=0.081226 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=19 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=16256 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.038299648e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none + [2026-09-21 08:31:46] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 221.3 | throughput per GPU (TFLOP/s/GPU): 145.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.078302E+00 | loss scale: 1.0 | grad norm: 16.575 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45138 payload_rows_remote=38690 dup_rows=20398 dup_rows_remote=16616 dedup_ratio=0.311249 router_tokens_per_expert=[127, 549, 646, 0, 0, 641, 1252, 346, 0, 858, 32, 2, 35, 47, 175, 120, 2, 908, 798, 1614, 34, 323, 286, 11, 99, 58, 109, 407, 1514, 622, 838, 1554, 512, 441, 434, 1384, 117, 1777, 835, 988, 906, 700, 1984, 70, 903, 761, 12, 215, 601, 668, 649, 88, 142, 828, 1660, 95, 946, 39, 199, 10, 9, 12, 755, 21] payload_rows_per_dst=[6448, 5948, 6096, 4764, 7208, 4468, 4724, 5482] dup_rows_per_dst=[3782, 2370, 2914, 1472, 5768, 854, 1486, 1752] combine_rows_per_src=[6448, 5948, 6096, 4764, 7208, 4468, 4724, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46430 payload_rows_remote=40630 dup_rows=19106 dup_rows_remote=16804 dedup_ratio=0.291534 router_tokens_per_expert=[183, 127, 595, 7, 1, 677, 931, 135, 5, 742, 22, 2, 13, 47, 315, 109, 6, 1087, 1137, 1662, 19, 335, 283, 3, 87, 3, 27, 211, 1100, 840, 1081, 1167, 849, 402, 622, 1113, 47, 1602, 553, 938, 1162, 396, 1634, 32, 737, 792, 15, 96, 1037, 545, 759, 33, 212, 1523, 2412, 75, 1349, 25, 257, 14, 23, 24, 508, 23] payload_rows_per_dst=[5504, 5800, 6460, 5174, 7224, 3708, 5774, 6786] dup_rows_per_dst=[2142, 2302, 2604, 1524, 5028, 428, 2594, 2484] combine_rows_per_src=[5504, 5800, 6460, 5174, 7224, 3708, 5774, 6786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46006 payload_rows_remote=40818 dup_rows=19530 dup_rows_remote=17894 dedup_ratio=0.298004 router_tokens_per_expert=[101, 189, 724, 4, 0, 733, 1258, 182, 0, 1424, 65, 0, 9, 8, 462, 112, 9, 1079, 816, 2099, 29, 63, 300, 1, 50, 26, 58, 187, 1071, 1060, 960, 1809, 380, 275, 411, 640, 187, 1522, 234, 818, 1143, 478, 2518, 37, 777, 952, 9, 53, 604, 474, 810, 65, 197, 1410, 1734, 59, 1213, 23, 349, 13, 11, 20, 452, 12] payload_rows_per_dst=[6512, 6918, 6238, 5188, 4472, 5710, 5126, 5842] dup_rows_per_dst=[3488, 4564, 2554, 1636, 1418, 1946, 2112, 1812] combine_rows_per_src=[6512, 6918, 6238, 5188, 4472, 5710, 5126, 5842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46496 payload_rows_remote=42562 dup_rows=19040 dup_rows_remote=18222 dedup_ratio=0.290527 router_tokens_per_expert=[281, 276, 763, 3, 4, 794, 765, 158, 0, 1337, 26, 6, 18, 19, 184, 158, 19, 1217, 958, 1715, 25, 273, 243, 17, 262, 10, 81, 270, 1347, 607, 823, 1571, 561, 186, 479, 631, 216, 1576, 220, 547, 1701, 380, 1653, 105, 904, 820, 8, 186, 944, 436, 641, 69, 222, 1645, 2062, 86, 1505, 29, 188, 6, 21, 23, 458, 30] payload_rows_per_dst=[5834, 5740, 6252, 5568, 3934, 6856, 5918, 6394] dup_rows_per_dst=[2602, 1670, 2682, 2026, 818, 3896, 3096, 2250] combine_rows_per_src=[5834, 5740, 6252, 5568, 3934, 6856, 5918, 6394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45946 payload_rows_remote=39750 dup_rows=19590 dup_rows_remote=16478 dedup_ratio=0.298920 router_tokens_per_expert=[128, 255, 748, 0, 0, 499, 1133, 398, 3, 1059, 21, 0, 6, 21, 432, 74, 15, 877, 1144, 1933, 33, 67, 243, 1, 83, 19, 59, 194, 1108, 1043, 1243, 1718, 946, 404, 351, 744, 226, 1304, 213, 1007, 1010, 726, 1902, 23, 892, 645, 11, 43, 738, 908, 617, 128, 172, 912, 1582, 14, 1174, 12, 343, 7, 7, 32, 1108, 10] payload_rows_per_dst=[4678, 5464, 6184, 6802, 4796, 6196, 5686, 6140] dup_rows_per_dst=[1644, 1572, 2442, 4132, 1498, 3112, 2780, 2410] combine_rows_per_src=[4678, 5464, 6184, 6802, 4796, 6196, 5686, 6140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46670 payload_rows_remote=40734 dup_rows=18866 dup_rows_remote=16856 dedup_ratio=0.287872 router_tokens_per_expert=[235, 167, 863, 2, 1, 422, 1126, 274, 1, 939, 25, 0, 10, 14, 146, 152, 2, 948, 1240, 1697, 31, 227, 266, 4, 70, 11, 123, 248, 969, 1037, 1052, 1671, 533, 325, 608, 895, 115, 1450, 298, 637, 1456, 696, 2282, 34, 750, 752, 15, 175, 887, 413, 765, 4, 172, 1530, 1844, 30, 1079, 24, 299, 8, 5, 14, 682, 18] payload_rows_per_dst=[4622, 5710, 4840, 6666, 6002, 6764, 6130, 5936] dup_rows_per_dst=[1558, 1428, 716, 3696, 2304, 3892, 3262, 2010] combine_rows_per_src=[4622, 5710, 4840, 6666, 6002, 6764, 6130, 5936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46216 payload_rows_remote=40626 dup_rows=19320 dup_rows_remote=17528 dedup_ratio=0.294800 router_tokens_per_expert=[227, 189, 872, 1, 2, 660, 891, 222, 1, 541, 31, 0, 17, 42, 579, 153, 1, 905, 702, 1716, 23, 202, 140, 2, 81, 32, 58, 235, 1090, 714, 1101, 1556, 430, 373, 582, 1350, 90, 1817, 579, 957, 1310, 612, 1945, 41, 867, 790, 34, 66, 784, 503, 652, 26, 274, 1211, 2038, 23, 1276, 11, 250, 4, 31, 15, 822, 19] payload_rows_per_dst=[6208, 6266, 5590, 5064, 6296, 5258, 5184, 6350] dup_rows_per_dst=[3032, 2972, 1792, 1558, 3864, 1758, 1762, 2582] combine_rows_per_src=[6208, 6266, 5590, 5064, 6296, 5258, 5184, 6350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158474240 dispatch_meta_bytes=221224 combine_payload_bytes=158474240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158695464 ingress_bytes=183640064 link_bytes=183640064 token_bytes=317169704 weight_movement_bytes=25165824 total_nvlink_bytes=342335528 weight_to_token_ratio=0.079345 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397288703e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166420480 dispatch_meta_bytes=229736 combine_payload_bytes=166420480 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=166650216 ingress_bytes=216752128 link_bytes=216752128 token_bytes=333070696 weight_movement_bytes=50331648 total_nvlink_bytes=383402344 weight_to_token_ratio=0.151114 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.649233251e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=167190528 dispatch_meta_bytes=234848 combine_payload_bytes=167190528 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=167425376 ingress_bytes=217522176 link_bytes=217522176 token_bytes=334615904 weight_movement_bytes=50331648 total_nvlink_bytes=384947552 weight_to_token_ratio=0.150416 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8704 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.655092426e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=174333952 dispatch_meta_bytes=243136 combine_payload_bytes=174333952 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=174577088 ingress_bytes=249831424 link_bytes=249831424 token_bytes=348911040 weight_movement_bytes=75497472 total_nvlink_bytes=424408512 weight_to_token_ratio=0.216380 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.900928472e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162816000 dispatch_meta_bytes=224912 combine_payload_bytes=162816000 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=163040912 ingress_bytes=238313472 link_bytes=238313472 token_bytes=325856912 weight_movement_bytes=75497472 total_nvlink_bytes=401354384 weight_to_token_ratio=0.231689 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.813290166e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45890 payload_rows_remote=39592 dup_rows=19646 dup_rows_remote=16042 dedup_ratio=0.299774 router_tokens_per_expert=[162, 149, 1127, 2, 0, 678, 1170, 453, 0, 951, 13, 0, 49, 16, 376, 58, 24, 901, 575, 1939, 44, 308, 294, 1, 47, 7, 58, 229, 910, 882, 828, 2172, 540, 307, 425, 1101, 160, 1566, 205, 816, 1710, 516, 1943, 98, 725, 638, 18, 74, 1029, 342, 622, 59, 161, 1594, 1586, 43, 1002, 42, 601, 9, 6, 26, 375, 6] payload_rows_per_dst=[5354, 5346, 5886, 6736, 3964, 6760, 6298, 5546] dup_rows_per_dst=[2128, 1466, 2286, 3530, 942, 3930, 3604, 1760] combine_rows_per_src=[5354, 5346, 5886, 6736, 3964, 6760, 6298, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166846464 dispatch_meta_bytes=230360 combine_payload_bytes=166846464 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=167076824 ingress_bytes=192012288 link_bytes=192012288 token_bytes=333923288 weight_movement_bytes=25165824 total_nvlink_bytes=359089112 weight_to_token_ratio=0.075364 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.460991654e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166404096 dispatch_meta_bytes=232616 combine_payload_bytes=166404096 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=166636712 ingress_bytes=216735744 link_bytes=216735744 token_bytes=333040808 weight_movement_bytes=50331648 total_nvlink_bytes=383372456 weight_to_token_ratio=0.151128 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.649108588e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162168832 dispatch_meta_bytes=222536 combine_payload_bytes=162168832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=162391368 ingress_bytes=212500480 link_bytes=212500480 token_bytes=324560200 weight_movement_bytes=50331648 total_nvlink_bytes=374891848 weight_to_token_ratio=0.155076 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.616883122e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56044 payload_rows_remote=48484 dup_rows=9492 dup_rows_remote=9290 dedup_ratio=0.144836 router_tokens_per_expert=[59, 0, 12, 0, 0, 906, 0, 76, 0, 2, 3687, 19, 0, 2, 171, 0, 0, 0, 3730, 33, 208, 35, 767, 0, 0, 6, 0, 0, 1, 0, 3785, 4, 0, 0, 3, 0, 209, 3345, 1144, 0, 4018, 0, 207, 0, 0, 0, 0, 18, 2, 7, 40, 6, 7, 1, 25, 0, 55, 0, 1076, 2986, 2521, 1, 3594, 0] payload_rows_per_dst=[8028, 7560, 1904, 7576, 7478, 8082, 7728, 7688] dup_rows_per_dst=[1538, 202, 182, 16, 1924, 404, 3462, 1764] combine_rows_per_src=[8028, 7560, 1904, 7576, 7478, 8082, 7728, 7688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53148 payload_rows_remote=45140 dup_rows=12388 dup_rows_remote=11714 dedup_ratio=0.189026 router_tokens_per_expert=[6, 0, 0, 0, 0, 414, 1, 0, 0, 1, 3884, 36, 0, 25, 13, 0, 0, 4, 3920, 28, 168, 318, 2606, 0, 0, 1259, 0, 13, 0, 54, 3599, 85, 1, 0, 0, 0, 299, 1012, 1686, 0, 3721, 0, 211, 6, 8, 0, 1, 0, 39, 6, 535, 8, 3, 0, 0, 1, 195, 5, 1499, 2441, 2017, 0, 2634, 6] payload_rows_per_dst=[8008, 7962, 5792, 7870, 5384, 4522, 6968, 6642] dup_rows_per_dst=[674, 1120, 724, 3756, 664, 282, 3132, 2036] combine_rows_per_src=[8008, 7962, 5792, 7870, 5384, 4522, 6968, 6642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57380 payload_rows_remote=49436 dup_rows=8156 dup_rows_remote=6008 dedup_ratio=0.124451 router_tokens_per_expert=[0, 0, 26, 0, 0, 358, 1, 89, 8, 3, 4079, 0, 7, 0, 7, 2, 4, 0, 4052, 1, 208, 3, 4015, 1, 0, 1541, 0, 0, 43, 0, 3462, 0, 0, 0, 0, 0, 1029, 3861, 617, 3, 4085, 6, 141, 1, 0, 0, 0, 1, 0, 0, 29, 0, 150, 0, 0, 1, 42, 0, 9, 1224, 2352, 0, 1306, 1] payload_rows_per_dst=[8174, 8180, 8048, 7944, 7924, 8182, 5628, 3300] dup_rows_per_dst=[878, 32, 416, 2148, 3096, 286, 954, 346] combine_rows_per_src=[8174, 8180, 8048, 7944, 7924, 8182, 5628, 3300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57874 payload_rows_remote=51226 dup_rows=7662 dup_rows_remote=6692 dedup_ratio=0.116913 router_tokens_per_expert=[2, 0, 3, 0, 0, 1182, 0, 87, 1, 0, 3998, 2, 3, 0, 6, 0, 0, 0, 4042, 0, 79, 91, 3871, 0, 0, 153, 0, 0, 0, 0, 2780, 1, 0, 0, 0, 0, 233, 2858, 718, 0, 4023, 0, 11, 0, 0, 0, 20, 0, 8, 11, 110, 777, 5, 0, 0, 1, 185, 0, 655, 2253, 3411, 0, 1188, 0] payload_rows_per_dst=[8174, 8000, 7762, 5804, 6648, 8054, 5608, 7824] dup_rows_per_dst=[2458, 20, 320, 64, 970, 54, 722, 3054] combine_rows_per_src=[8174, 8000, 7762, 5804, 6648, 8054, 5608, 7824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59504 payload_rows_remote=51322 dup_rows=6032 dup_rows_remote=5950 dedup_ratio=0.092041 router_tokens_per_expert=[0, 0, 0, 0, 0, 1135, 0, 2, 0, 19, 4055, 31, 0, 0, 11, 0, 0, 0, 3906, 21, 302, 202, 3965, 0, 0, 854, 0, 0, 0, 5, 3162, 3, 0, 0, 0, 0, 40, 2591, 1387, 0, 4083, 0, 49, 0, 0, 0, 0, 0, 97, 0, 31, 5, 0, 0, 99, 0, 38, 6, 181, 2284, 478, 0, 3726, 0] payload_rows_per_dst=[8082, 8154, 8082, 7450, 6778, 8182, 4948, 7828] dup_rows_per_dst=[2004, 78, 898, 598, 1258, 82, 84, 1030] combine_rows_per_src=[8082, 8154, 8082, 7450, 6778, 8182, 4948, 7828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56920 payload_rows_remote=51044 dup_rows=8616 dup_rows_remote=8412 dedup_ratio=0.131470 router_tokens_per_expert=[0, 2, 0, 0, 0, 190, 0, 0, 0, 0, 3943, 1, 0, 0, 56, 0, 0, 0, 4054, 14, 0, 198, 2824, 4, 0, 171, 0, 0, 5, 0, 3284, 0, 0, 0, 0, 0, 592, 2776, 546, 0, 4094, 0, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 120, 26, 0, 22, 0, 205, 1, 419, 3681, 1954, 0, 2224, 0] payload_rows_per_dst=[8118, 7936, 5876, 6790, 6766, 8190, 8028, 5216] dup_rows_per_dst=[374, 64, 204, 130, 1062, 2722, 3578, 482] combine_rows_per_src=[8118, 7936, 5876, 6790, 6766, 8190, 8028, 5216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52376 payload_rows_remote=46122 dup_rows=13160 dup_rows_remote=12324 dedup_ratio=0.200806 router_tokens_per_expert=[12, 0, 6, 0, 0, 19, 0, 22, 0, 0, 4002, 3, 0, 0, 186, 0, 0, 0, 3988, 1, 16, 18, 3573, 1, 0, 13, 3, 0, 0, 1, 3885, 0, 0, 0, 0, 0, 1401, 2006, 1555, 15, 3893, 0, 230, 0, 0, 0, 3, 0, 17, 0, 1197, 242, 0, 0, 0, 7, 231, 1, 1, 2924, 2866, 0, 430, 0] payload_rows_per_dst=[184, 8062, 8144, 7782, 7300, 7818, 6832, 6254] dup_rows_per_dst=[40, 320, 6944, 22, 2654, 402, 1942, 836] combine_rows_per_src=[184, 8062, 8144, 7782, 7300, 7818, 6832, 6254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=198590464 dispatch_meta_bytes=231096 combine_payload_bytes=198590464 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198821560 ingress_bytes=223756288 link_bytes=223756288 token_bytes=397412024 weight_movement_bytes=25165824 total_nvlink_bytes=422577848 weight_to_token_ratio=0.063324 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.702526816e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=184893440 dispatch_meta_bytes=227416 combine_payload_bytes=184893440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=185120856 ingress_bytes=210059264 link_bytes=210059264 token_bytes=370014296 weight_movement_bytes=25165824 total_nvlink_bytes=395180120 weight_to_token_ratio=0.068013 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598308289e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=202489856 dispatch_meta_bytes=221776 combine_payload_bytes=202489856 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202711632 ingress_bytes=227655680 link_bytes=227655680 token_bytes=405201488 weight_movement_bytes=25165824 total_nvlink_bytes=430367312 weight_to_token_ratio=0.062107 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.732196683e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=209821696 dispatch_meta_bytes=231672 combine_payload_bytes=209821696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210053368 ingress_bytes=234987520 link_bytes=234987520 token_bytes=419875064 weight_movement_bytes=25165824 total_nvlink_bytes=445040888 weight_to_token_ratio=0.059936 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787983514e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=210214912 dispatch_meta_bytes=229088 combine_payload_bytes=210214912 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=210444000 ingress_bytes=336044032 link_bytes=336044032 token_bytes=420658912 weight_movement_bytes=125829120 total_nvlink_bytes=546488032 weight_to_token_ratio=0.299124 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.556906806e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=209076224 dispatch_meta_bytes=237824 combine_payload_bytes=209076224 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=209314048 ingress_bytes=259407872 link_bytes=259407872 token_bytes=418390272 weight_movement_bytes=50331648 total_nvlink_bytes=468721920 weight_to_token_ratio=0.120298 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.973794177e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=188915712 dispatch_meta_bytes=233784 combine_payload_bytes=188915712 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=189149496 ingress_bytes=264413184 link_bytes=264413184 token_bytes=378065208 weight_movement_bytes=75497472 total_nvlink_bytes=453562680 weight_to_token_ratio=0.199694 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.011878818e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57448 payload_rows_remote=51744 dup_rows=8088 dup_rows_remote=7870 dedup_ratio=0.123413 router_tokens_per_expert=[6, 0, 1, 0, 0, 943, 0, 9, 1, 3, 3912, 0, 0, 0, 7, 0, 0, 0, 3442, 0, 1049, 29, 4042, 0, 0, 7, 0, 0, 2, 24, 3722, 38, 0, 0, 0, 0, 211, 2272, 331, 1, 4053, 0, 83, 0, 0, 0, 1, 0, 119, 1, 120, 7, 1, 0, 0, 0, 13, 0, 1092, 2713, 2414, 0, 2099, 0] payload_rows_per_dst=[7240, 7830, 8100, 7552, 5332, 8108, 5704, 7582] dup_rows_per_dst=[1562, 16, 2140, 34, 298, 166, 218, 3654] combine_rows_per_src=[7240, 7830, 8100, 7552, 5332, 8108, 5704, 7582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52944 payload_rows_remote=45578 dup_rows=12592 dup_rows_remote=11398 dedup_ratio=0.192139 router_tokens_per_expert=[0, 3, 0, 0, 3919, 1, 3775, 0, 212, 0, 0, 4091, 1, 3443, 0, 624, 2000, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 1, 6, 0, 2300, 11, 0, 2, 0, 0, 3, 4, 0, 1, 0, 1898, 1, 0, 0, 1, 0, 4083, 0, 0, 4081, 3, 0, 0, 0, 5, 0, 2210, 2, 0, 0] payload_rows_per_dst=[7906, 7366, 4088, 8192, 4626, 8182, 8164, 4420] dup_rows_per_dst=[7490, 1194, 0, 90, 6, 3792, 6, 14] combine_rows_per_src=[7906, 7366, 4088, 8192, 4626, 8182, 8164, 4420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59060 payload_rows_remote=50880 dup_rows=6476 dup_rows_remote=6470 dedup_ratio=0.098816 router_tokens_per_expert=[0, 1, 0, 1, 4052, 0, 4095, 0, 17, 0, 5, 4087, 4, 3772, 0, 2444, 2537, 215, 3, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4, 0, 0, 0, 3, 3117, 6, 0, 3, 0, 1, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 7, 4068, 279, 0, 3533, 59, 0, 0, 6, 0, 1, 383, 2, 28, 0] payload_rows_per_dst=[8188, 7958, 5324, 8180, 6256, 8184, 7536, 7434] dup_rows_per_dst=[772, 4288, 424, 6, 4, 18, 220, 744] combine_rows_per_src=[8188, 7958, 5324, 8180, 6256, 8184, 7536, 7434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45710 payload_rows_remote=37518 dup_rows=19826 dup_rows_remote=9372 dedup_ratio=0.302521 router_tokens_per_expert=[0, 2, 0, 0, 3477, 0, 4096, 0, 108, 0, 0, 4090, 1, 3630, 1, 3161, 1043, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 1315, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 34, 0, 0, 1, 0, 7, 3928, 410, 0, 4086, 1213, 0, 0, 0, 0, 0, 2138, 0, 3, 0] payload_rows_per_dst=[8192, 8158, 2104, 8180, 2638, 7916, 4240, 4282] dup_rows_per_dst=[10454, 8800, 4, 16, 2, 10, 540, 0] combine_rows_per_src=[8192, 8158, 2104, 8180, 2638, 7916, 4240, 4282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56550 payload_rows_remote=53286 dup_rows=8986 dup_rows_remote=8104 dedup_ratio=0.137115 router_tokens_per_expert=[0, 0, 0, 0, 4082, 0, 4085, 0, 286, 0, 0, 4058, 0, 4055, 0, 3283, 858, 21, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1021, 16, 0, 0, 25, 3, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 53, 4047, 1, 3, 3711, 1148, 0, 0, 11, 0, 0, 1664, 61, 31, 0] payload_rows_per_dst=[6148, 6800, 8152, 8118, 3264, 8094, 7782, 8192] dup_rows_per_dst=[0, 338, 1722, 22, 882, 460, 2050, 3512] combine_rows_per_src=[6148, 6800, 8152, 8118, 3264, 8094, 7782, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=211943424 dispatch_meta_bytes=238456 combine_payload_bytes=211943424 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=212181880 ingress_bytes=262275072 link_bytes=262275072 token_bytes=424125304 weight_movement_bytes=50331648 total_nvlink_bytes=474456952 weight_to_token_ratio=0.118672 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=20608 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.995610256e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50698 payload_rows_remote=42510 dup_rows=14838 dup_rows_remote=14836 dedup_ratio=0.226410 router_tokens_per_expert=[0, 0, 0, 1, 4089, 0, 4092, 0, 745, 0, 1, 4091, 0, 3556, 0, 513, 3643, 1, 10, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 2689, 0, 0, 0, 36, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 74, 259, 4092, 0, 0, 3575, 931, 0, 0, 0, 0, 0, 351, 0, 0, 1] payload_rows_per_dst=[2, 2180, 7960, 8192, 8188, 8188, 7804, 8184] dup_rows_per_dst=[0, 338, 6464, 14, 5442, 2, 1874, 704] combine_rows_per_src=[2, 2180, 7960, 8192, 8188, 8188, 7804, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54074 payload_rows_remote=50732 dup_rows=11462 dup_rows_remote=11440 dedup_ratio=0.174896 router_tokens_per_expert=[0, 0, 0, 0, 4072, 0, 4095, 0, 0, 0, 3, 4074, 0, 4028, 0, 1118, 1625, 0, 5, 42, 10, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 1984, 0, 0, 0, 3, 0, 0, 1, 0, 0, 15, 188, 1, 0, 0, 9, 7, 4082, 8, 0, 3449, 399, 0, 6, 0, 0, 0, 3417, 119, 4, 1] payload_rows_per_dst=[8190, 8144, 3342, 8150, 3978, 8172, 7200, 6898] dup_rows_per_dst=[8144, 2154, 22, 0, 0, 402, 544, 196] combine_rows_per_src=[8190, 8144, 3342, 8150, 3978, 8172, 7200, 6898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=186687488 dispatch_meta_bytes=227904 combine_payload_bytes=186687488 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=186915392 ingress_bytes=287350784 link_bytes=287350784 token_bytes=373602880 weight_movement_bytes=100663296 total_nvlink_bytes=474266176 weight_to_token_ratio=0.269439 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.186407451e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47578 payload_rows_remote=39402 dup_rows=17958 dup_rows_remote=17648 dedup_ratio=0.274017 router_tokens_per_expert=[2, 0, 23, 0, 4018, 0, 4074, 0, 7, 0, 0, 4092, 0, 3967, 0, 2770, 3825, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3177, 4, 0, 0, 8, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3, 3, 4083, 0, 23, 2431, 72, 0, 2, 13, 0, 0, 139, 4, 11, 0] payload_rows_per_dst=[50, 5378, 8192, 8186, 8116, 3866, 5614, 8176] dup_rows_per_dst=[0, 4, 7398, 186, 6298, 8, 3754, 310] combine_rows_per_src=[50, 5378, 8192, 8186, 8116, 3866, 5614, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=208404480 dispatch_meta_bytes=229400 combine_payload_bytes=208404480 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=208633880 ingress_bytes=258736128 link_bytes=258736128 token_bytes=417038360 weight_movement_bytes=50331648 total_nvlink_bytes=467370008 weight_to_token_ratio=0.120688 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.968682981e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=153673728 dispatch_meta_bytes=187560 combine_payload_bytes=153673728 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=153861288 ingress_bytes=229171200 link_bytes=229171200 token_bytes=307535016 weight_movement_bytes=75497472 total_nvlink_bytes=383032488 weight_to_token_ratio=0.245492 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.743728039e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=218259456 dispatch_meta_bytes=245560 combine_payload_bytes=218259456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=218505016 ingress_bytes=243425280 link_bytes=243425280 token_bytes=436764472 weight_movement_bytes=25165824 total_nvlink_bytes=461930296 weight_to_token_ratio=0.057619 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.852185118e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=174120960 dispatch_meta_bytes=229384 combine_payload_bytes=174120960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=174350344 ingress_bytes=199286784 link_bytes=199286784 token_bytes=348471304 weight_movement_bytes=25165824 total_nvlink_bytes=373637128 weight_to_token_ratio=0.072218 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.516342163e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=207798272 dispatch_meta_bytes=248688 combine_payload_bytes=207798272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=208046960 ingress_bytes=232964096 link_bytes=232964096 token_bytes=415845232 weight_movement_bytes=25165824 total_nvlink_bytes=441011056 weight_to_token_ratio=0.060517 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16896 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.772587595e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=161390592 dispatch_meta_bytes=228200 combine_payload_bytes=161390592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=161618792 ingress_bytes=186556416 link_bytes=186556416 token_bytes=323009384 weight_movement_bytes=25165824 total_nvlink_bytes=348175208 weight_to_token_ratio=0.077911 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.419478772e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50444 payload_rows_remote=42914 dup_rows=15092 dup_rows_remote=14414 dedup_ratio=0.230286 router_tokens_per_expert=[0, 0, 0, 0, 3926, 0, 3785, 0, 25, 0, 0, 3990, 0, 3805, 0, 1193, 1359, 3, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 4085, 14, 0, 3706, 384, 0, 0, 8, 0, 0, 2428, 1, 13, 0] payload_rows_per_dst=[0, 2422, 7996, 7982, 8188, 8170, 7530, 8156] dup_rows_per_dst=[0, 14, 2344, 22, 7648, 72, 678, 4314] combine_rows_per_src=[0, 2422, 7996, 7982, 8188, 8170, 7530, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61466 payload_rows_remote=53278 dup_rows=4070 dup_rows_remote=1920 dedup_ratio=0.062103 router_tokens_per_expert=[0, 1, 0, 1, 0, 4095, 0, 16, 0, 4092, 0, 0, 0, 1074, 1, 2, 0, 0, 4055, 2483, 0, 5, 0, 0, 1, 0, 1, 1, 0, 7, 0, 4070, 0, 7, 3812, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 12, 0, 1545, 0, 0, 57, 0, 3336, 0, 0, 0, 0, 3, 6, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 4970, 8144, 7624, 8178, 8042, 8128] dup_rows_per_dst=[34, 2150, 6, 16, 14, 16, 1834, 0] combine_rows_per_src=[8192, 8188, 4970, 8144, 7624, 8178, 8042, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58744 payload_rows_remote=50570 dup_rows=6792 dup_rows_remote=6786 dedup_ratio=0.103638 router_tokens_per_expert=[0, 0, 0, 0, 1, 4094, 0, 0, 0, 3738, 14, 0, 0, 1646, 0, 3, 0, 0, 4084, 3358, 0, 0, 0, 0, 0, 0, 10, 0, 0, 6, 0, 4074, 0, 0, 1648, 0, 0, 2, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 4, 0, 2549, 0, 0, 0, 0, 3430, 0, 0, 8, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8176, 6716, 8174, 3300, 8192, 7812, 8184] dup_rows_per_dst=[0, 2626, 0, 6, 0, 14, 4146, 0] combine_rows_per_src=[8190, 8176, 6716, 8174, 3300, 8192, 7812, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62270 payload_rows_remote=55446 dup_rows=3266 dup_rows_remote=2612 dedup_ratio=0.049835 router_tokens_per_expert=[0, 0, 0, 0, 0, 4084, 0, 0, 0, 2173, 1, 0, 0, 4038, 0, 1, 0, 0, 4096, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4096, 0, 0, 2664, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 6, 0, 337, 0, 0, 74, 0, 3095, 0, 0, 0, 0, 0, 93, 0, 0, 0] payload_rows_per_dst=[8168, 8124, 7808, 8192, 6824, 8192, 6770, 8192] dup_rows_per_dst=[0, 2152, 0, 8, 654, 24, 242, 186] combine_rows_per_src=[8168, 8124, 7808, 8192, 6824, 8192, 6770, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58790 payload_rows_remote=50598 dup_rows=6746 dup_rows_remote=6730 dedup_ratio=0.102936 router_tokens_per_expert=[0, 0, 0, 0, 4, 4096, 0, 0, 0, 4075, 0, 1, 0, 2059, 0, 17, 0, 0, 3954, 3887, 0, 13, 0, 1, 0, 0, 0, 16, 2, 37, 0, 4076, 0, 0, 2546, 3, 0, 0, 0, 0, 0, 1, 3957, 0, 0, 0, 0, 71, 0, 3, 15, 2, 14, 3, 3899, 1, 5, 0, 7, 0, 0, 3, 0, 0] payload_rows_per_dst=[8192, 8184, 7798, 8178, 5092, 8034, 5696, 7616] dup_rows_per_dst=[16, 5330, 322, 76, 6, 966, 26, 4] combine_rows_per_src=[8192, 8184, 7798, 8178, 5092, 8034, 5696, 7616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=175775744 dispatch_meta_bytes=229312 combine_payload_bytes=175775744 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=176005056 ingress_bytes=251273216 link_bytes=251273216 token_bytes=351780800 weight_movement_bytes=75497472 total_nvlink_bytes=427278272 weight_to_token_ratio=0.214615 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=25344 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.911898844e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57024 payload_rows_remote=49670 dup_rows=8512 dup_rows_remote=8512 dedup_ratio=0.129883 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 3989, 119, 0, 0, 3436, 0, 0, 0, 0, 4096, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4091, 0, 0, 3848, 0, 0, 2, 0, 0, 1, 0, 3676, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 3228, 0, 0, 0, 0, 2, 8, 0, 0, 0] payload_rows_per_dst=[8192, 3904, 6534, 8190, 8190, 7354, 6468, 8192] dup_rows_per_dst=[0, 232, 754, 6, 7488, 0, 12, 20] combine_rows_per_src=[8192, 3904, 6534, 8190, 8190, 7354, 6468, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56976 payload_rows_remote=48796 dup_rows=8560 dup_rows_remote=8560 dedup_ratio=0.130615 router_tokens_per_expert=[1, 0, 4, 1, 0, 4095, 0, 0, 0, 4089, 62, 0, 0, 4056, 0, 1, 0, 0, 4083, 4089, 1, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4093, 0, 0, 152, 0, 0, 3, 0, 0, 0, 0, 4056, 3, 0, 0, 0, 0, 0, 176, 29, 0, 16, 0, 3597, 0, 0, 0, 1, 2, 146, 8, 0, 0] payload_rows_per_dst=[8192, 8192, 8180, 8186, 308, 8118, 7610, 8190] dup_rows_per_dst=[10, 8224, 0, 8, 2, 0, 26, 290] combine_rows_per_src=[8192, 8192, 8180, 8186, 308, 8118, 7610, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=218226688 dispatch_meta_bytes=220792 combine_payload_bytes=218226688 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=218447480 ingress_bytes=293724160 link_bytes=293724160 token_bytes=436674168 weight_movement_bytes=75497472 total_nvlink_bytes=512171640 weight_to_token_ratio=0.172892 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=17152 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.234901478e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56406 payload_rows_remote=48242 dup_rows=9130 dup_rows_remote=9084 dedup_ratio=0.139313 router_tokens_per_expert=[0, 0, 2, 0, 0, 4088, 0, 4, 0, 3435, 0, 0, 0, 1358, 0, 0, 5, 0, 4082, 4070, 0, 0, 0, 2, 0, 0, 0, 0, 0, 13, 0, 4095, 0, 0, 3522, 0, 1, 0, 0, 13, 0, 1, 4092, 25, 0, 0, 0, 3, 2, 0, 0, 0, 227, 3, 3702, 0, 0, 0, 0, 0, 23, 0, 0, 0] payload_rows_per_dst=[8186, 0, 8192, 8132, 7908, 8190, 7634, 8164] dup_rows_per_dst=[2, 0, 2678, 26, 6034, 52, 292, 46] combine_rows_per_src=[8186, 0, 8192, 8132, 7908, 8190, 7634, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=207134720 dispatch_meta_bytes=229424 combine_payload_bytes=207134720 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=207364144 ingress_bytes=257466368 link_bytes=257466368 token_bytes=414498864 weight_movement_bytes=50331648 total_nvlink_bytes=464830512 weight_to_token_ratio=0.121428 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.959021575e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=227106816 dispatch_meta_bytes=232232 combine_payload_bytes=227106816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=227339048 ingress_bytes=252272640 link_bytes=252272640 token_bytes=454445864 weight_movement_bytes=25165824 total_nvlink_bytes=479611688 weight_to_token_ratio=0.055377 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=20480 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.919503305e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=207249408 dispatch_meta_bytes=229312 combine_payload_bytes=207249408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=207478720 ingress_bytes=232415232 link_bytes=232415232 token_bytes=414728128 weight_movement_bytes=25165824 total_nvlink_bytes=439893952 weight_to_token_ratio=0.060680 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.768411375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=203448320 dispatch_meta_bytes=232728 combine_payload_bytes=203448320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203681048 ingress_bytes=228614144 link_bytes=228614144 token_bytes=407129368 weight_movement_bytes=25165824 total_nvlink_bytes=432295192 weight_to_token_ratio=0.061813 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.739489487e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=199868416 dispatch_meta_bytes=229424 combine_payload_bytes=199868416 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=200097840 ingress_bytes=250200064 link_bytes=250200064 token_bytes=399966256 weight_movement_bytes=50331648 total_nvlink_bytes=450297904 weight_to_token_ratio=0.125840 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=26752 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.903733397e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=197599232 dispatch_meta_bytes=229304 combine_payload_bytes=197599232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197828536 ingress_bytes=222765056 link_bytes=222765056 token_bytes=395427768 weight_movement_bytes=25165824 total_nvlink_bytes=420593592 weight_to_token_ratio=0.063642 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.694984685e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56050 payload_rows_remote=48800 dup_rows=9486 dup_rows_remote=7810 dedup_ratio=0.144745 router_tokens_per_expert=[0, 0, 0, 0, 0, 4081, 0, 0, 0, 3878, 0, 0, 0, 3084, 0, 27, 0, 0, 4095, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2168, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 3, 0, 1257, 0, 0, 79, 0, 3127, 0, 0, 0, 0, 0, 67, 0, 0, 0] payload_rows_per_dst=[8162, 54, 7916, 8192, 8092, 8192, 7250, 8192] dup_rows_per_dst=[0, 0, 3662, 2, 4000, 14, 1676, 132] combine_rows_per_src=[8162, 54, 7916, 8192, 8092, 8192, 7250, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58062 payload_rows_remote=49870 dup_rows=7474 dup_rows_remote=7450 dedup_ratio=0.114044 router_tokens_per_expert=[0, 0, 0, 0, 2230, 0, 2, 4096, 4091, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1199, 0, 1, 0, 0, 2, 0, 0, 301, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 4082, 4094, 0, 4095, 390, 0, 0, 0, 0, 0, 4089, 47, 4014, 0, 4, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8156, 8172, 780, 8190, 8190] dup_rows_per_dst=[4464, 24, 2414, 480, 4, 0, 82, 6] combine_rows_per_src=[8192, 8192, 8190, 8156, 8172, 780, 8190, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63994 payload_rows_remote=55802 dup_rows=1542 dup_rows_remote=1034 dedup_ratio=0.023529 router_tokens_per_expert=[0, 17, 0, 1, 365, 0, 0, 3882, 4093, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 9, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4087, 4073, 0, 4002, 3654, 2, 0, 0, 0, 0, 4050, 10, 4096, 0, 139, 0, 1, 0, 0, 0, 0, 1] payload_rows_per_dst=[7810, 8186, 8048, 8192, 8174, 7316, 8116, 8152] dup_rows_per_dst=[720, 0, 26, 508, 0, 8, 4, 276] combine_rows_per_src=[7810, 8186, 8048, 8192, 8174, 7316, 8116, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62236 payload_rows_remote=54044 dup_rows=3300 dup_rows_remote=3300 dedup_ratio=0.050354 router_tokens_per_expert=[0, 1, 0, 0, 1777, 0, 0, 3901, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4093, 2576, 0, 0, 0, 0, 0, 3974, 0, 4095, 0, 55, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8192, 8190, 8190, 8192, 5152, 7948, 8192] dup_rows_per_dst=[3178, 0, 10, 0, 0, 2, 0, 110] combine_rows_per_src=[8180, 8192, 8190, 8190, 8192, 5152, 7948, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=199884800 dispatch_meta_bytes=226440 combine_payload_bytes=199884800 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=200111240 ingress_bytes=275382272 link_bytes=275382272 token_bytes=399996040 weight_movement_bytes=75497472 total_nvlink_bytes=475493512 weight_to_token_ratio=0.188746 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.095340903e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61322 payload_rows_remote=53130 dup_rows=4214 dup_rows_remote=1242 dedup_ratio=0.064301 router_tokens_per_expert=[0, 0, 8, 2, 1274, 5, 0, 4055, 4085, 8, 0, 31, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 3, 3, 2, 0, 0, 15, 0, 0, 0, 74, 0, 0, 0, 1, 0, 0, 0, 3, 0, 41, 4077, 4055, 0, 4066, 2689, 0, 2, 0, 1, 0, 3828, 61, 4061, 2, 309, 1, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8148, 8156, 8184, 5948, 7304, 7200] dup_rows_per_dst=[2972, 58, 6, 148, 66, 396, 8, 560] combine_rows_per_src=[8192, 8190, 8148, 8156, 8184, 5948, 7304, 7200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65040 payload_rows_remote=57136 dup_rows=496 dup_rows_remote=496 dedup_ratio=0.007568 router_tokens_per_expert=[0, 0, 0, 0, 141, 2, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 3987, 3951, 0, 0, 0, 0, 0, 4084, 11, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 7998, 8192, 8192, 7904, 8180, 8192] dup_rows_per_dst=[286, 0, 0, 200, 0, 0, 10, 0] combine_rows_per_src=[8192, 8190, 7998, 8192, 8192, 7904, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64372 payload_rows_remote=56180 dup_rows=1164 dup_rows_remote=1152 dedup_ratio=0.017761 router_tokens_per_expert=[0, 1, 0, 0, 432, 0, 0, 3973, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4081, 4095, 4, 4096, 3648, 0, 0, 0, 18, 0, 4096, 1, 4086, 0, 7, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7958, 8180, 8192, 8192, 8162, 7304, 8192, 8192] dup_rows_per_dst=[854, 0, 12, 242, 0, 2, 38, 16] combine_rows_per_src=[7958, 8180, 8192, 8192, 8162, 7304, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53636 payload_rows_remote=45448 dup_rows=11900 dup_rows_remote=11862 dedup_ratio=0.181580 router_tokens_per_expert=[0, 0, 0, 0, 211, 0, 0, 3939, 4088, 7, 0, 0, 57, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 95, 0, 3, 0, 0, 1, 0, 0, 12, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4089, 3867, 0, 0, 0, 0, 0, 4092, 0, 4092, 0, 19, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[422, 8016, 5612, 6830, 8192, 8192, 8184, 8188] dup_rows_per_dst=[0, 116, 136, 28, 6, 10042, 1534, 38] combine_rows_per_src=[422, 8016, 5612, 6830, 8192, 8192, 8184, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=204267520 dispatch_meta_bytes=229280 combine_payload_bytes=204267520 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=204496800 ingress_bytes=229433344 link_bytes=229433344 token_bytes=408764320 weight_movement_bytes=25165824 total_nvlink_bytes=433930144 weight_to_token_ratio=0.061566 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.745722652e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=228564992 dispatch_meta_bytes=227344 combine_payload_bytes=228564992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=228792336 ingress_bytes=253730816 link_bytes=253730816 token_bytes=457357328 weight_movement_bytes=25165824 total_nvlink_bytes=482523152 weight_to_token_ratio=0.055024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.930598340e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=221364224 dispatch_meta_bytes=229376 combine_payload_bytes=221364224 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=221593600 ingress_bytes=271695872 link_bytes=271695872 token_bytes=442957824 weight_movement_bytes=50331648 total_nvlink_bytes=493289472 weight_to_token_ratio=0.113626 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.067291659e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=217620480 dispatch_meta_bytes=217488 combine_payload_bytes=217620480 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=217837968 ingress_bytes=267952128 link_bytes=267952128 token_bytes=435458448 weight_movement_bytes=50331648 total_nvlink_bytes=485790096 weight_to_token_ratio=0.115583 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25600 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.038806093e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=234029056 dispatch_meta_bytes=230528 combine_payload_bytes=234029056 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=234259584 ingress_bytes=359858176 link_bytes=359858176 token_bytes=468288640 weight_movement_bytes=125829120 total_nvlink_bytes=594117760 weight_to_token_ratio=0.268700 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.738104926e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=230113280 dispatch_meta_bytes=229328 combine_payload_bytes=230113280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230342608 ingress_bytes=255279104 link_bytes=255279104 token_bytes=460455888 weight_movement_bytes=25165824 total_nvlink_bytes=485621712 weight_to_token_ratio=0.054654 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.942379023e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=186155008 dispatch_meta_bytes=229240 combine_payload_bytes=186155008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186384248 ingress_bytes=211320832 link_bytes=211320832 token_bytes=372539256 weight_movement_bytes=25165824 total_nvlink_bytes=397705080 weight_to_token_ratio=0.067552 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.607907364e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60214 payload_rows_remote=52468 dup_rows=5322 dup_rows_remote=5322 dedup_ratio=0.081207 router_tokens_per_expert=[0, 0, 0, 0, 532, 0, 0, 4031, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4064, 3740, 0, 0, 0, 0, 0, 3873, 0, 4009, 0, 16, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3634, 8192, 8162, 8018, 8192, 8078, 7746, 8192] dup_rows_per_dst=[482, 0, 392, 4, 0, 4412, 0, 32] combine_rows_per_src=[3634, 8192, 8162, 8018, 8192, 8078, 7746, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63666 payload_rows_remote=56682 dup_rows=1870 dup_rows_remote=1648 dedup_ratio=0.028534 router_tokens_per_expert=[0, 0, 0, 191, 0, 368, 0, 0, 1, 0, 4, 4092, 0, 3, 0, 0, 0, 4090, 65, 32, 0, 0, 0, 1, 2774, 803, 23, 0, 0, 3, 0, 0, 4089, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 4079, 44, 0, 152, 0, 0, 4074, 0, 0, 0, 0, 0, 3794, 0, 0, 14, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8190, 6984, 8178, 8192, 8148, 7594] dup_rows_per_dst=[1074, 8, 186, 222, 0, 358, 0, 22] combine_rows_per_src=[8188, 8192, 8190, 6984, 8178, 8192, 8148, 7594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61852 payload_rows_remote=53680 dup_rows=3684 dup_rows_remote=3684 dedup_ratio=0.056213 router_tokens_per_expert=[0, 0, 42, 125, 0, 969, 0, 9, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4054, 191, 35, 0, 0, 41, 0, 197, 3753, 0, 0, 0, 15, 0, 0, 3096, 0, 0, 0, 2, 3, 3988, 0, 0, 0, 1, 0, 4085, 402, 0, 32, 4, 1, 4077, 0, 0, 1, 2, 1, 3556, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 8166, 7666, 6202, 8180, 8162, 7112] dup_rows_per_dst=[2074, 0, 476, 264, 0, 860, 10, 0] combine_rows_per_src=[8192, 8172, 8166, 7666, 6202, 8180, 8162, 7112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=214908928 dispatch_meta_bytes=231160 combine_payload_bytes=214908928 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=215140088 ingress_bytes=290406400 link_bytes=290406400 token_bytes=430049016 weight_movement_bytes=75497472 total_nvlink_bytes=505546488 weight_to_token_ratio=0.175556 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.209657158e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62150 payload_rows_remote=54030 dup_rows=3386 dup_rows_remote=3386 dedup_ratio=0.051666 router_tokens_per_expert=[0, 0, 0, 1, 0, 54, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 3997, 439, 0, 0, 0, 0, 0, 2347, 2622, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4000, 2, 0, 75, 0, 0, 4086, 0, 0, 0, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8088, 7586, 8120, 8012, 8172, 5788] dup_rows_per_dst=[108, 0, 784, 2352, 0, 142, 0, 0] combine_rows_per_src=[8192, 8192, 8088, 7586, 8120, 8012, 8172, 5788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55090 payload_rows_remote=46922 dup_rows=10446 dup_rows_remote=2574 dedup_ratio=0.159393 router_tokens_per_expert=[0, 3, 1, 0, 0, 3935, 1, 0, 0, 2, 0, 4087, 6, 48, 0, 0, 0, 3107, 0, 25, 0, 0, 0, 219, 734, 1980, 0, 0, 3, 0, 0, 0, 3550, 0, 8, 0, 0, 0, 4080, 0, 0, 39, 0, 0, 4090, 0, 0, 45, 27, 2, 4004, 0, 0, 0, 5, 1, 2710, 0, 0, 40, 0, 0, 16, 0] payload_rows_per_dst=[8168, 8192, 7862, 4736, 7116, 5500, 8062, 5454] dup_rows_per_dst=[7872, 94, 1650, 698, 0, 38, 16, 78] combine_rows_per_src=[8168, 8192, 7862, 4736, 7116, 5500, 8062, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56054 payload_rows_remote=47864 dup_rows=9482 dup_rows_remote=9414 dedup_ratio=0.144684 router_tokens_per_expert=[0, 0, 0, 0, 0, 2783, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 3263, 38, 1, 0, 0, 2, 0, 3063, 814, 0, 0, 0, 4, 0, 0, 3335, 0, 0, 0, 0, 7, 4031, 0, 0, 1, 0, 0, 4094, 24, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 2330, 0, 0, 782, 0, 0, 0, 0] payload_rows_per_dst=[6664, 8180, 6586, 6460, 4488, 8190, 8192, 7294] dup_rows_per_dst=[4906, 0, 22, 1302, 4, 68, 0, 3180] combine_rows_per_src=[6664, 8180, 6586, 6460, 4488, 8190, 8192, 7294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58848 payload_rows_remote=50924 dup_rows=6688 dup_rows_remote=5142 dedup_ratio=0.102051 router_tokens_per_expert=[0, 0, 0, 13, 0, 768, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 3761, 938, 4, 0, 0, 0, 32, 2583, 3136, 0, 2, 0, 2, 0, 0, 3911, 0, 1, 0, 0, 8, 4018, 0, 0, 0, 0, 0, 4096, 3, 0, 6, 0, 0, 4096, 0, 0, 1, 0, 0, 1258, 0, 0, 48, 0, 0, 0, 0] payload_rows_per_dst=[8178, 8166, 7924, 7748, 7840, 8192, 8192, 2608] dup_rows_per_dst=[1420, 0, 1546, 3698, 0, 18, 2, 4] combine_rows_per_src=[8178, 8166, 7924, 7748, 7840, 8192, 8192, 2608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48504 payload_rows_remote=40312 dup_rows=17032 dup_rows_remote=8978 dedup_ratio=0.259888 router_tokens_per_expert=[0, 0, 0, 0, 0, 15, 0, 2, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 2995, 0, 0, 0, 0, 13, 0, 1337, 3410, 0, 0, 0, 0, 0, 0, 4068, 0, 2, 0, 0, 7, 4096, 0, 1, 0, 0, 0, 4090, 0, 4, 485, 0, 0, 4092, 0, 2, 0, 0, 0, 3857, 0, 0, 196, 0, 0, 0, 2] payload_rows_per_dst=[34, 8148, 6010, 1552, 8192, 8188, 8188, 8192] dup_rows_per_dst=[0, 0, 6, 0, 6638, 2134, 200, 8054] combine_rows_per_src=[34, 8148, 6010, 1552, 8192, 8188, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=232169472 dispatch_meta_bytes=233320 combine_payload_bytes=232169472 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=232402792 ingress_bytes=332832768 link_bytes=332832768 token_bytes=464572264 weight_movement_bytes=100663296 total_nvlink_bytes=565235560 weight_to_token_ratio=0.216680 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.532472797e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=219873280 dispatch_meta_bytes=229456 combine_payload_bytes=219873280 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=220102736 ingress_bytes=245039104 link_bytes=245039104 token_bytes=439976016 weight_movement_bytes=25165824 total_nvlink_bytes=465141840 weight_to_token_ratio=0.057198 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.864464454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=221306880 dispatch_meta_bytes=229664 combine_payload_bytes=221306880 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=221536544 ingress_bytes=296804352 link_bytes=296804352 token_bytes=442843424 weight_movement_bytes=75497472 total_nvlink_bytes=518340896 weight_to_token_ratio=0.170483 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=19968 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.258338180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=192192512 dispatch_meta_bytes=197984 combine_payload_bytes=192192512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=192390496 ingress_bytes=217358336 link_bytes=217358336 token_bytes=384583008 weight_movement_bytes=25165824 total_nvlink_bytes=409748832 weight_to_token_ratio=0.065437 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23296 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.653845793e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=196050944 dispatch_meta_bytes=229112 combine_payload_bytes=196050944 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=196280056 ingress_bytes=246382592 link_bytes=246382592 token_bytes=392331000 weight_movement_bytes=50331648 total_nvlink_bytes=442662648 weight_to_token_ratio=0.128289 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.874686846e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=208584704 dispatch_meta_bytes=224264 combine_payload_bytes=208584704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=208808968 ingress_bytes=233750528 link_bytes=233750528 token_bytes=417393672 weight_movement_bytes=25165824 total_nvlink_bytes=442559496 weight_to_token_ratio=0.060293 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=29312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.778571434e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=165117952 dispatch_meta_bytes=197160 combine_payload_bytes=165117952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=165315112 ingress_bytes=190283776 link_bytes=190283776 token_bytes=330433064 weight_movement_bytes=25165824 total_nvlink_bytes=355598888 weight_to_token_ratio=0.076160 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.447839675e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56860 payload_rows_remote=48668 dup_rows=8676 dup_rows_remote=8676 dedup_ratio=0.132385 router_tokens_per_expert=[0, 0, 0, 2, 0, 231, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4005, 15, 0, 0, 0, 0, 0, 3781, 3636, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3710, 7, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 956, 0, 0, 27, 0, 0, 0, 0] payload_rows_per_dst=[466, 8192, 8014, 8160, 8192, 7452, 8192, 8192] dup_rows_per_dst=[0, 0, 26, 6108, 566, 12, 0, 1964] combine_rows_per_src=[466, 8192, 8014, 8160, 8192, 7452, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64968 payload_rows_remote=56948 dup_rows=568 dup_rows_remote=568 dedup_ratio=0.008667 router_tokens_per_expert=[3930, 0, 22, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 76, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4063, 4096, 0, 0, 0, 4010, 0, 1, 0, 0, 4096, 0, 5, 20, 0, 1, 42, 0, 0, 0, 0, 18, 0, 0, 1, 0, 0, 4053, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7894, 8192, 8192, 8020, 8192, 8192, 8108, 8178] dup_rows_per_dst=[10, 100, 4, 0, 52, 122, 280, 0] combine_rows_per_src=[7894, 8192, 8192, 8020, 8192, 8192, 8108, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63768 payload_rows_remote=55576 dup_rows=1768 dup_rows_remote=1724 dedup_ratio=0.026978 router_tokens_per_expert=[4074, 0, 134, 0, 3991, 4048, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 4091, 1, 48, 0, 4096, 0, 1, 0, 0, 4095, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8162, 8192, 8182, 8192, 8192, 6770, 8094, 7984] dup_rows_per_dst=[1584, 44, 2, 98, 40, 0, 0, 0] combine_rows_per_src=[8162, 8192, 8182, 8192, 8192, 6770, 8094, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=199344128 dispatch_meta_bytes=229376 combine_payload_bytes=199344128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199573504 ingress_bytes=224509952 link_bytes=224509952 token_bytes=398917632 weight_movement_bytes=25165824 total_nvlink_bytes=424083456 weight_to_token_ratio=0.063085 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.708261328e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63116 payload_rows_remote=54924 dup_rows=2420 dup_rows_remote=2214 dedup_ratio=0.036926 router_tokens_per_expert=[3089, 0, 2, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4075, 0, 0, 0, 0, 4096, 0, 103, 0, 0, 0, 982, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6182, 8192, 8192, 8150, 8192, 8186, 7832, 8190] dup_rows_per_dst=[0, 2, 0, 0, 206, 1966, 246, 0] combine_rows_per_src=[6182, 8192, 8192, 8150, 8192, 8186, 7832, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52276 payload_rows_remote=44096 dup_rows=13260 dup_rows_remote=3580 dedup_ratio=0.202332 router_tokens_per_expert=[3977, 0, 0, 0, 4079, 4087, 0, 0, 4, 0, 0, 13, 0, 0, 0, 11, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4000, 4040, 0, 0, 0, 4054, 27, 0, 0, 0, 4084, 0, 404, 0, 21, 0, 187, 6, 7, 0, 1, 0, 0, 0, 25, 0, 0, 3589, 82, 0, 0, 0, 6, 0, 0, 53, 3, 6] payload_rows_per_dst=[8180, 8026, 8140, 8040, 4136, 424, 7138, 8192] dup_rows_per_dst=[9680, 30, 14, 3368, 44, 20, 2, 102] combine_rows_per_src=[8180, 8026, 8140, 8040, 4136, 424, 7138, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64064 payload_rows_remote=56060 dup_rows=1472 dup_rows_remote=1470 dedup_ratio=0.022461 router_tokens_per_expert=[3845, 0, 0, 0, 4096, 4002, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 4096, 4096, 0, 3, 0, 3791, 0, 0, 0, 0, 4096, 0, 404, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7690, 8192, 8192, 7588, 8192, 8004, 8014, 8192] dup_rows_per_dst=[0, 380, 58, 0, 808, 2, 224, 0] combine_rows_per_src=[7690, 8192, 8192, 7588, 8192, 8004, 8014, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62086 payload_rows_remote=53908 dup_rows=3450 dup_rows_remote=3450 dedup_ratio=0.052643 router_tokens_per_expert=[3635, 0, 6, 0, 4071, 3921, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4089, 0, 0, 2, 2915, 0, 0, 0, 0, 4096, 0, 43, 0, 0, 0, 1788, 0, 0, 0, 2, 3, 0, 0, 0, 3, 0, 4094, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7278, 8110, 8178, 5830, 8192, 8166, 8190, 8142] dup_rows_per_dst=[4, 2, 0, 4, 86, 3262, 92, 0] combine_rows_per_src=[7278, 8110, 8178, 5830, 8192, 8166, 8190, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=227639296 dispatch_meta_bytes=229200 combine_payload_bytes=227639296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=227868496 ingress_bytes=252805120 link_bytes=252805120 token_bytes=455507792 weight_movement_bytes=25165824 total_nvlink_bytes=480673616 weight_to_token_ratio=0.055248 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.923554863e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=233259008 dispatch_meta_bytes=230064 combine_payload_bytes=233259008 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=233489072 ingress_bytes=333922304 link_bytes=333922304 token_bytes=466748080 weight_movement_bytes=100663296 total_nvlink_bytes=567411376 weight_to_token_ratio=0.215669 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.540762908e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57902 payload_rows_remote=49728 dup_rows=7634 dup_rows_remote=7444 dedup_ratio=0.116486 router_tokens_per_expert=[607, 0, 0, 1, 4086, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 3982, 4093, 0, 0, 2, 4081, 0, 7, 0, 0, 4092, 1, 1372, 0, 0, 0, 1889, 0, 0, 0, 0, 0, 2, 2, 0, 0, 3, 3996, 442, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[1026, 7892, 8186, 8168, 8192, 8188, 8076, 8174] dup_rows_per_dst=[0, 6, 2, 92, 2752, 3778, 814, 190] combine_rows_per_src=[1026, 7892, 8186, 8168, 8192, 8188, 8076, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=224968704 dispatch_meta_bytes=228552 combine_payload_bytes=224968704 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=225197256 ingress_bytes=300466176 link_bytes=300466176 token_bytes=450165960 weight_movement_bytes=75497472 total_nvlink_bytes=525663432 weight_to_token_ratio=0.167710 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=30208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.286200430e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=180617216 dispatch_meta_bytes=190704 combine_payload_bytes=180617216 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=180807920 ingress_bytes=281280512 link_bytes=281280512 token_bytes=361425136 weight_movement_bytes=100663296 total_nvlink_bytes=462088432 weight_to_token_ratio=0.278518 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.140219695e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=229621760 dispatch_meta_bytes=230120 combine_payload_bytes=229621760 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229851880 ingress_bytes=254787584 link_bytes=254787584 token_bytes=459473640 weight_movement_bytes=25165824 total_nvlink_bytes=484639464 weight_to_token_ratio=0.054771 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.938639123e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=220807168 dispatch_meta_bytes=229432 combine_payload_bytes=220807168 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=221036600 ingress_bytes=271138816 link_bytes=271138816 token_bytes=441843768 weight_movement_bytes=50331648 total_nvlink_bytes=492175416 weight_to_token_ratio=0.113913 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.063053106e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=203685888 dispatch_meta_bytes=228688 combine_payload_bytes=203685888 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=203914576 ingress_bytes=254017536 link_bytes=254017536 token_bytes=407600464 weight_movement_bytes=50331648 total_nvlink_bytes=457932112 weight_to_token_ratio=0.123483 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.932779948e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64846 payload_rows_remote=56770 dup_rows=690 dup_rows_remote=340 dedup_ratio=0.010529 router_tokens_per_expert=[3922, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4085, 4096, 0, 0, 0, 3993, 0, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 127, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7846, 8170, 8192, 7986, 8192, 8192, 8076, 8192] dup_rows_per_dst=[0, 6, 2, 0, 68, 264, 350, 0] combine_rows_per_src=[7846, 8170, 8192, 7986, 8192, 8192, 8076, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58756 payload_rows_remote=50716 dup_rows=6780 dup_rows_remote=6780 dedup_ratio=0.103455 router_tokens_per_expert=[0, 0, 0, 4078, 0, 0, 118, 0, 0, 0, 4018, 0, 0, 0, 1, 1, 0, 0, 0, 0, 43, 0, 0, 4089, 1, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 4074, 4093, 191, 3, 5, 0, 8, 2986, 0, 0, 0, 684, 3209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4, 0, 864, 165] payload_rows_per_dst=[8156, 8040, 8192, 7812, 8152, 8192, 8186, 2026] dup_rows_per_dst=[236, 0, 72, 4656, 394, 1380, 0, 42] combine_rows_per_src=[8156, 8040, 8192, 7812, 8152, 8192, 8186, 2026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57124 payload_rows_remote=49210 dup_rows=8412 dup_rows_remote=8302 dedup_ratio=0.128357 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4087, 392, 0, 0, 0, 0, 3668, 0, 0, 0, 22, 4055, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 73, 48] payload_rows_per_dst=[8192, 8084, 8162, 7914, 8164, 8192, 8174, 242] dup_rows_per_dst=[42, 0, 0, 110, 716, 7466, 78, 0] combine_rows_per_src=[8192, 8084, 8162, 7914, 8164, 8192, 8174, 242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=232529920 dispatch_meta_bytes=228440 combine_payload_bytes=232529920 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=232758360 ingress_bytes=257695744 link_bytes=257695744 token_bytes=465288280 weight_movement_bytes=25165824 total_nvlink_bytes=490454104 weight_to_token_ratio=0.054087 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.960766861e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55478 payload_rows_remote=47300 dup_rows=10058 dup_rows_remote=9952 dedup_ratio=0.153473 router_tokens_per_expert=[0, 4, 0, 4061, 12, 0, 0, 0, 0, 2, 2068, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 6, 4087, 0, 0, 0, 0, 0, 0, 13, 6, 0, 0, 0, 4009, 3901, 4, 22, 0, 0, 0, 4089, 5, 0, 8, 369, 3813, 12, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4073, 0, 0, 0, 1494, 690] payload_rows_per_dst=[8178, 7418, 8064, 8192, 3754, 8192, 7826, 3854] dup_rows_per_dst=[106, 1042, 12, 7650, 0, 718, 0, 530] combine_rows_per_src=[8178, 7418, 8064, 8192, 3754, 8192, 7826, 3854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55374 payload_rows_remote=47186 dup_rows=10162 dup_rows_remote=3746 dedup_ratio=0.155060 router_tokens_per_expert=[0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 4096, 3211, 0, 0, 0, 0, 828, 0, 0, 0, 0, 1452, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 2472, 243] payload_rows_per_dst=[8192, 8170, 8192, 1658, 8188, 4966, 8192, 7816] dup_rows_per_dst=[0, 0, 0, 0, 6416, 292, 2, 3452] combine_rows_per_src=[8192, 8170, 8192, 1658, 8188, 4966, 8192, 7816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62042 payload_rows_remote=54110 dup_rows=3494 dup_rows_remote=3432 dedup_ratio=0.053314 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 45, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4082, 4094, 3, 0, 0, 0, 0, 2514, 0, 1, 0, 31, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1632, 26] payload_rows_per_dst=[8192, 8152, 8192, 5030, 8164, 7932, 8188, 8192] dup_rows_per_dst=[90, 0, 20, 0, 6, 62, 0, 3316] combine_rows_per_src=[8192, 8152, 8192, 5030, 8164, 7932, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57868 payload_rows_remote=49676 dup_rows=7668 dup_rows_remote=7638 dedup_ratio=0.117004 router_tokens_per_expert=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 50, 0, 0, 0, 0, 0, 14, 1, 0, 4096, 0, 0, 0, 4, 39, 0, 0, 0, 0, 1, 0, 4049, 4082, 3020, 0, 1, 0, 0, 3293, 0, 0, 0, 170, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 393, 425] payload_rows_per_dst=[8184, 8190, 8192, 7178, 8132, 8192, 8164, 1636] dup_rows_per_dst=[0, 84, 30, 1204, 6010, 340, 0, 0] combine_rows_per_src=[8184, 8190, 8192, 7178, 8132, 8192, 8164, 1636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=207732736 dispatch_meta_bytes=229984 combine_payload_bytes=207732736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=207962720 ingress_bytes=232898560 link_bytes=232898560 token_bytes=415695456 weight_movement_bytes=25165824 total_nvlink_bytes=440861280 weight_to_token_ratio=0.060539 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.772088942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=201564160 dispatch_meta_bytes=230048 combine_payload_bytes=201564160 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=201794208 ingress_bytes=251895808 link_bytes=251895808 token_bytes=403358368 weight_movement_bytes=50331648 total_nvlink_bytes=453690016 weight_to_token_ratio=0.124781 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.916636049e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53490 payload_rows_remote=45302 dup_rows=12046 dup_rows_remote=98 dedup_ratio=0.183807 router_tokens_per_expert=[2, 0, 3, 4049, 19, 2, 14, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 32, 0, 0, 0, 3, 4084, 4010, 177, 0, 0, 0, 0, 3260, 0, 16, 0, 0, 2906, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 1343, 547] payload_rows_per_dst=[7704, 8188, 8192, 6586, 792, 5812, 8028, 8188] dup_rows_per_dst=[40, 0, 22, 2, 2, 32, 0, 11948] combine_rows_per_src=[7704, 8188, 8192, 6586, 792, 5812, 8028, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=193740800 dispatch_meta_bytes=229008 combine_payload_bytes=193740800 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=193969808 ingress_bytes=244072448 link_bytes=244072448 token_bytes=387710608 weight_movement_bytes=50331648 total_nvlink_bytes=438042256 weight_to_token_ratio=0.129818 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.857109319e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=193273856 dispatch_meta_bytes=203728 combine_payload_bytes=193273856 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=193477584 ingress_bytes=319102976 link_bytes=319102976 token_bytes=386751440 weight_movement_bytes=125829120 total_nvlink_bytes=512580560 weight_to_token_ratio=0.325349 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.428004944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=203472896 dispatch_meta_bytes=229256 combine_payload_bytes=203472896 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=203702152 ingress_bytes=228638720 link_bytes=228638720 token_bytes=407175048 weight_movement_bytes=25165824 total_nvlink_bytes=432340872 weight_to_token_ratio=0.061806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.739676482e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=221634560 dispatch_meta_bytes=230168 combine_payload_bytes=221634560 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=221864728 ingress_bytes=297132032 link_bytes=297132032 token_bytes=443499288 weight_movement_bytes=75497472 total_nvlink_bytes=518996760 weight_to_token_ratio=0.170231 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17536 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.260831446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57826 payload_rows_remote=49636 dup_rows=7710 dup_rows_remote=7202 dedup_ratio=0.117645 router_tokens_per_expert=[0, 0, 0, 4094, 18, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3916, 4094, 2441, 0, 0, 0, 0, 3770, 0, 0, 0, 345, 1539, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 254, 17] payload_rows_per_dst=[8190, 8168, 8192, 7544, 5622, 3728, 8190, 8192] dup_rows_per_dst=[34, 0, 2, 0, 1024, 40, 508, 6102] combine_rows_per_src=[8190, 8168, 8192, 7544, 5622, 3728, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=185556992 dispatch_meta_bytes=181600 combine_payload_bytes=185556992 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=185738592 ingress_bytes=235888640 link_bytes=235888640 token_bytes=371295584 weight_movement_bytes=50331648 total_nvlink_bytes=421627232 weight_to_token_ratio=0.135557 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.794839996e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57294 payload_rows_remote=57144 dup_rows=8242 dup_rows_remote=8186 dedup_ratio=0.125763 router_tokens_per_expert=[0, 0, 0, 3954, 0, 0, 4081, 15, 42, 30, 0, 0, 28, 0, 0, 3, 31, 0, 0, 4095, 0, 19, 0, 4095, 0, 0, 4067, 0, 77, 0, 1, 0, 0, 2, 4022, 0, 4085, 0, 0, 0, 12, 1, 0, 13, 0, 0, 0, 0, 0, 4079, 0, 1, 6, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 150, 8192, 8166, 8048, 8190, 8172, 8192] dup_rows_per_dst=[7916, 56, 98, 124, 0, 32, 2, 14] combine_rows_per_src=[8184, 150, 8192, 8166, 8048, 8190, 8172, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61056 payload_rows_remote=52864 dup_rows=4480 dup_rows_remote=4468 dedup_ratio=0.068359 router_tokens_per_expert=[0, 0, 0, 2162, 0, 0, 4093, 4, 0, 0, 0, 0, 2136, 0, 0, 0, 94, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 3793, 1, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4332, 8190, 8192, 8192, 7588, 8186, 8186, 8190] dup_rows_per_dst=[0, 4268, 188, 12, 0, 4, 8, 0] combine_rows_per_src=[4332, 8190, 8192, 8192, 7588, 8186, 8186, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=203309056 dispatch_meta_bytes=227352 combine_payload_bytes=203309056 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=203536408 ingress_bytes=253640704 link_bytes=253640704 token_bytes=406845464 weight_movement_bytes=50331648 total_nvlink_bytes=457177112 weight_to_token_ratio=0.123712 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.929912692e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56874 payload_rows_remote=48714 dup_rows=8662 dup_rows_remote=968 dedup_ratio=0.132172 router_tokens_per_expert=[0, 0, 0, 3847, 0, 0, 4080, 0, 0, 0, 0, 7, 45, 0, 14, 0, 47, 2, 0, 4083, 0, 0, 0, 4040, 0, 5, 4088, 0, 103, 13, 0, 4, 0, 0, 4024, 151, 4020, 0, 7, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 5, 0, 0, 0, 1, 9, 0, 0, 0, 13, 0] payload_rows_per_dst=[8160, 118, 8180, 7818, 8192, 8040, 8188, 8178] dup_rows_per_dst=[7694, 14, 600, 26, 314, 0, 0, 14] combine_rows_per_src=[8160, 118, 8180, 7818, 8192, 8040, 8188, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64406 payload_rows_remote=56642 dup_rows=1130 dup_rows_remote=1130 dedup_ratio=0.017242 router_tokens_per_expert=[0, 0, 0, 4085, 0, 0, 4096, 0, 0, 1, 0, 0, 562, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 8192, 8192, 8192, 7764, 8188, 7516, 8192] dup_rows_per_dst=[0, 1126, 2, 0, 0, 0, 2, 0] combine_rows_per_src=[8170, 8192, 8192, 8192, 7764, 8188, 7516, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63060 payload_rows_remote=54868 dup_rows=2476 dup_rows_remote=2212 dedup_ratio=0.037781 router_tokens_per_expert=[0, 1, 0, 3517, 0, 0, 4095, 0, 0, 0, 0, 0, 1138, 0, 0, 0, 132, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 4090, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7992, 7302, 8192, 8192, 8190, 8184, 6816, 8192] dup_rows_per_dst=[1068, 1140, 264, 4, 0, 0, 0, 0] combine_rows_per_src=[7992, 7302, 8192, 8192, 8190, 8184, 6816, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58150 payload_rows_remote=49958 dup_rows=7386 dup_rows_remote=7384 dedup_ratio=0.112701 router_tokens_per_expert=[0, 0, 0, 416, 0, 0, 4096, 0, 0, 0, 0, 0, 3692, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[832, 8192, 8192, 8192, 8166, 8192, 8192, 8192] dup_rows_per_dst=[0, 7384, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[832, 8192, 8192, 8192, 8166, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=234061824 dispatch_meta_bytes=261320 combine_payload_bytes=234061824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234323144 ingress_bytes=259227648 link_bytes=259227648 token_bytes=468384968 weight_movement_bytes=25165824 total_nvlink_bytes=493550792 weight_to_token_ratio=0.053729 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.972422880e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=216530944 dispatch_meta_bytes=229328 combine_payload_bytes=216530944 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=216760272 ingress_bytes=266862592 link_bytes=266862592 token_bytes=433291216 weight_movement_bytes=50331648 total_nvlink_bytes=483622864 weight_to_token_ratio=0.116161 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.030515982e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61510 payload_rows_remote=53330 dup_rows=4026 dup_rows_remote=4026 dedup_ratio=0.061432 router_tokens_per_expert=[0, 28, 3, 4055, 4, 2, 4092, 1, 11, 0, 0, 1, 135, 0, 0, 0, 0, 2, 0, 4087, 0, 0, 0, 4093, 0, 1, 4093, 0, 0, 4, 0, 1, 0, 0, 4025, 0, 4091, 3, 0, 0, 0, 0, 0, 15, 4, 0, 0, 0, 0, 4012, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5624, 8190, 7052, 8192, 8056, 8188, 8028, 8180] dup_rows_per_dst=[70, 288, 0, 234, 1028, 142, 2264, 0] combine_rows_per_src=[5624, 8190, 7052, 8192, 8056, 8188, 8028, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=199532544 dispatch_meta_bytes=198728 combine_payload_bytes=199532544 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=199731272 ingress_bytes=275030016 link_bytes=275030016 token_bytes=399263816 weight_movement_bytes=75497472 total_nvlink_bytes=474761288 weight_to_token_ratio=0.189092 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23552 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.092660642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=232005632 dispatch_meta_bytes=231088 combine_payload_bytes=232005632 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=232236720 ingress_bytes=307503104 link_bytes=307503104 token_bytes=464242352 weight_movement_bytes=75497472 total_nvlink_bytes=539739824 weight_to_token_ratio=0.162625 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.339743321e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=224739328 dispatch_meta_bytes=228320 combine_payload_bytes=224739328 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=224967648 ingress_bytes=300236800 link_bytes=300236800 token_bytes=449706976 weight_movement_bytes=75497472 total_nvlink_bytes=525204448 weight_to_token_ratio=0.167881 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.284455144e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=204627968 dispatch_meta_bytes=229368 combine_payload_bytes=204627968 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=204857336 ingress_bytes=254959616 link_bytes=254959616 token_bytes=409485304 weight_movement_bytes=50331648 total_nvlink_bytes=459816952 weight_to_token_ratio=0.122914 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.939948088e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62800 payload_rows_remote=54640 dup_rows=2736 dup_rows_remote=2734 dedup_ratio=0.041748 router_tokens_per_expert=[0, 0, 0, 2747, 0, 0, 4096, 0, 0, 0, 0, 0, 1367, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5494, 8192, 8192, 8192, 8190, 8188, 8160, 8192] dup_rows_per_dst=[0, 2734, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[5494, 8192, 8192, 8192, 8190, 8188, 8160, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=218439680 dispatch_meta_bytes=229424 combine_payload_bytes=218439680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=218669104 ingress_bytes=243605504 link_bytes=243605504 token_bytes=437108784 weight_movement_bytes=25165824 total_nvlink_bytes=462274608 weight_to_token_ratio=0.057573 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.853556415e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58138 payload_rows_remote=49950 dup_rows=7398 dup_rows_remote=7398 dedup_ratio=0.112885 router_tokens_per_expert=[0, 1, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4086, 0, 0, 134, 14, 0, 0, 0, 4096, 0, 0, 358, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 0, 4075, 2, 0, 0, 0, 4068, 0, 1, 0, 0, 0, 0, 329, 4092, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8152, 8188, 8184, 8192, 8192, 6636, 2410, 8184] dup_rows_per_dst=[12, 0, 284, 7046, 0, 0, 56, 0] combine_rows_per_src=[8152, 8188, 8184, 8192, 8192, 6636, 2410, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61098 payload_rows_remote=52906 dup_rows=4438 dup_rows_remote=1440 dedup_ratio=0.067719 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 10, 0, 52, 13, 0, 0, 0, 4096, 0, 0, 1499, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 4083, 0, 0, 0, 0, 3934, 0, 46, 0, 0, 0, 0, 672, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8192, 8192, 8192, 8170, 3974, 8014, 8192] dup_rows_per_dst=[0, 0, 150, 2998, 0, 0, 1290, 0] combine_rows_per_src=[8172, 8192, 8192, 8192, 8170, 3974, 8014, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=223805440 dispatch_meta_bytes=229496 combine_payload_bytes=223805440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=224034936 ingress_bytes=248971264 link_bytes=248971264 token_bytes=447840376 weight_movement_bytes=25165824 total_nvlink_bytes=473006200 weight_to_token_ratio=0.056194 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.894383649e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56202 payload_rows_remote=48036 dup_rows=9334 dup_rows_remote=9170 dedup_ratio=0.142426 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4027, 0, 3, 4081, 1, 7, 53, 0, 0, 0, 0, 4072, 53, 4, 362, 0, 0, 0, 0, 3936, 19, 0, 0, 0, 0, 1, 0, 66, 3361, 0, 4082, 0, 9, 0, 0, 4084, 1, 10, 32, 8, 0, 0, 414, 4024, 0, 6, 0, 0, 13, 9, 12] payload_rows_per_dst=[8166, 8124, 8118, 8192, 7910, 6698, 920, 8074] dup_rows_per_dst=[164, 30, 108, 8958, 2, 8, 10, 54] combine_rows_per_src=[8166, 8124, 8118, 8192, 7910, 6698, 920, 8074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=204595200 dispatch_meta_bytes=229392 combine_payload_bytes=204595200 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=204824592 ingress_bytes=229761024 link_bytes=229761024 token_bytes=409419792 weight_movement_bytes=25165824 total_nvlink_bytes=434585616 weight_to_token_ratio=0.061467 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.748215918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59132 payload_rows_remote=51010 dup_rows=6404 dup_rows_remote=6404 dedup_ratio=0.097717 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 459, 0, 0, 0, 0, 4096, 0, 0, 1793, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 820, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 4466, 8122, 5592, 8184, 8192] dup_rows_per_dst=[0, 0, 926, 1922, 0, 1922, 1634, 0] combine_rows_per_src=[8192, 8192, 8192, 4466, 8122, 5592, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60336 payload_rows_remote=52144 dup_rows=5200 dup_rows_remote=5060 dedup_ratio=0.079346 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 69, 0, 0, 0, 1, 4094, 0, 0, 124, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 1513, 0, 4095, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 2448, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8164, 3026, 8188, 8192] dup_rows_per_dst=[0, 0, 140, 244, 0, 0, 4816, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8164, 3026, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57778 payload_rows_remote=49586 dup_rows=7758 dup_rows_remote=940 dedup_ratio=0.118378 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 6, 0, 170, 0, 0, 0, 0, 4096, 0, 0, 272, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 0, 4056, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 280, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8116, 8186, 8192, 544, 8164, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 352, 0, 0, 6818, 588, 0] combine_rows_per_src=[8116, 8186, 8192, 544, 8164, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=216702976 dispatch_meta_bytes=217384 combine_payload_bytes=216702976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=216920360 ingress_bytes=267034624 link_bytes=267034624 token_bytes=433623336 weight_movement_bytes=50331648 total_nvlink_bytes=483954984 weight_to_token_ratio=0.116072 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.031824947e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56988 payload_rows_remote=48860 dup_rows=8548 dup_rows_remote=8412 dedup_ratio=0.130432 router_tokens_per_expert=[0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 27, 0, 0, 4093, 0, 0, 4092, 1, 2, 425, 0, 0, 0, 0, 4091, 0, 4, 1397, 0, 1, 0, 0, 4000, 6, 0, 0, 0, 0, 0, 2, 0, 1045, 0, 4087, 0, 0, 0, 0, 4088, 0, 0, 1, 2, 0, 0, 1330, 4061, 0, 0, 0, 1, 2, 2, 3] payload_rows_per_dst=[8180, 8190, 5310, 2802, 8006, 8182, 8190, 8128] dup_rows_per_dst=[4, 50, 778, 2, 936, 2090, 4552, 136] combine_rows_per_src=[8180, 8190, 5310, 2802, 8006, 8182, 8190, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=196755456 dispatch_meta_bytes=228824 combine_payload_bytes=196755456 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=196984280 ingress_bytes=247087104 link_bytes=247087104 token_bytes=393739736 weight_movement_bytes=50331648 total_nvlink_bytes=444071384 weight_to_token_ratio=0.127830 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.880047368e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=208936960 dispatch_meta_bytes=229656 combine_payload_bytes=208936960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=209166616 ingress_bytes=234102784 link_bytes=234102784 token_bytes=418103576 weight_movement_bytes=25165824 total_nvlink_bytes=443269400 weight_to_token_ratio=0.060190 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781251695e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=213581824 dispatch_meta_bytes=228816 combine_payload_bytes=213581824 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=213810640 ingress_bytes=314245120 link_bytes=314245120 token_bytes=427392464 weight_movement_bytes=100663296 total_nvlink_bytes=528055760 weight_to_token_ratio=0.235529 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.391042273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=203104256 dispatch_meta_bytes=202104 combine_payload_bytes=203104256 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=203306360 ingress_bytes=278601728 link_bytes=278601728 token_bytes=406410616 weight_movement_bytes=75497472 total_nvlink_bytes=481908088 weight_to_token_ratio=0.185766 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.119837244e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60798 payload_rows_remote=52612 dup_rows=4738 dup_rows_remote=4432 dedup_ratio=0.072296 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 113, 0, 0, 0, 0, 4096, 0, 0, 1893, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 0, 4094, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 179, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 3786, 7870, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 226, 0, 0, 4206, 306, 0] combine_rows_per_src=[8188, 8192, 8192, 3786, 7870, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=200130560 dispatch_meta_bytes=229088 combine_payload_bytes=200130560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200359648 ingress_bytes=225296384 link_bytes=225296384 token_bytes=400490208 weight_movement_bytes=25165824 total_nvlink_bytes=425656032 weight_to_token_ratio=0.062838 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.714245167e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65174 payload_rows_remote=56984 dup_rows=362 dup_rows_remote=338 dedup_ratio=0.005524 router_tokens_per_expert=[0, 4096, 0, 0, 0, 3942, 0, 0, 0, 4094, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4072, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 106, 0, 0, 0, 4096, 0, 0, 1, 2, 0, 1, 0, 0, 0] payload_rows_per_dst=[7884, 8190, 8192, 8146, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 24, 86, 0, 34, 0, 210, 8] combine_rows_per_src=[7884, 8190, 8192, 8146, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64508 payload_rows_remote=56376 dup_rows=1028 dup_rows_remote=1028 dedup_ratio=0.015686 router_tokens_per_expert=[0, 4096, 2, 0, 0, 3622, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 24, 0, 0, 4066, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 6, 0, 0, 0, 4083, 0, 11, 1, 0, 0, 4096, 0, 0, 177, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7248, 8192, 8192, 8132, 8192, 8192, 8168, 8192] dup_rows_per_dst=[0, 0, 590, 0, 50, 12, 22, 354] combine_rows_per_src=[7248, 8192, 8192, 8132, 8192, 8192, 8168, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=215498752 dispatch_meta_bytes=228176 combine_payload_bytes=215498752 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=215726928 ingress_bytes=265830400 link_bytes=265830400 token_bytes=431225680 weight_movement_bytes=50331648 total_nvlink_bytes=481557328 weight_to_token_ratio=0.116718 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25344 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.022662194e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61522 payload_rows_remote=53764 dup_rows=4014 dup_rows_remote=934 dedup_ratio=0.061249 router_tokens_per_expert=[0, 4040, 0, 4, 0, 3300, 0, 17, 20, 3934, 0, 0, 0, 0, 0, 13, 0, 0, 0, 33, 13, 0, 0, 0, 3703, 4083, 4056, 0, 6, 0, 0, 7, 2, 0, 0, 41, 0, 6, 1, 0, 0, 8, 11, 11, 20, 0, 0, 2, 4045, 0, 1048, 0, 0, 0, 4061, 0, 0, 113, 61, 0, 46, 9, 0, 54] payload_rows_per_dst=[7758, 7932, 8142, 7538, 8140, 8144, 7078, 6790] dup_rows_per_dst=[3080, 236, 62, 16, 4, 82, 350, 184] combine_rows_per_src=[7758, 7932, 8142, 7538, 8140, 8144, 7078, 6790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233406464 dispatch_meta_bytes=229288 combine_payload_bytes=233406464 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233635752 ingress_bytes=258572288 link_bytes=258572288 token_bytes=467042216 weight_movement_bytes=25165824 total_nvlink_bytes=492208040 weight_to_token_ratio=0.053883 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.967436348e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230916096 dispatch_meta_bytes=229616 combine_payload_bytes=230916096 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=231145712 ingress_bytes=306413568 link_bytes=306413568 token_bytes=462061808 weight_movement_bytes=75497472 total_nvlink_bytes=537559280 weight_to_token_ratio=0.163393 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.331453211e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65528 payload_rows_remote=57336 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000122 router_tokens_per_expert=[0, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8184] dup_rows_per_dst=[0, 0, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65404 payload_rows_remote=57300 dup_rows=132 dup_rows_remote=132 dedup_ratio=0.002014 router_tokens_per_expert=[0, 4096, 1, 0, 0, 4078, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 51, 0, 0, 0, 4095, 0, 0, 11, 0, 10, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8190, 8104, 8192, 8192, 8190, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 0, 20, 0, 70, 42] combine_rows_per_src=[8158, 8190, 8104, 8192, 8192, 8190, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64278 payload_rows_remote=56086 dup_rows=1258 dup_rows_remote=966 dedup_ratio=0.019196 router_tokens_per_expert=[0, 4087, 0, 0, 0, 3476, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 1, 0, 0, 4094, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 4096, 0, 0, 435, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6952, 8190, 8192, 8188, 8192, 8192, 8192, 8180] dup_rows_per_dst=[0, 0, 70, 0, 0, 292, 32, 864] combine_rows_per_src=[6952, 8190, 8192, 8188, 8192, 8192, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64100 payload_rows_remote=55952 dup_rows=1436 dup_rows_remote=1422 dedup_ratio=0.021912 router_tokens_per_expert=[0, 4066, 2, 0, 0, 4090, 0, 0, 2, 4081, 0, 0, 4, 8, 0, 4, 3, 0, 0, 0, 10, 0, 21, 0, 4096, 4067, 4090, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4088, 0, 22, 0, 0, 0, 4088, 0, 0, 4, 7, 2, 1, 1, 0, 0] payload_rows_per_dst=[8180, 8184, 7232, 8192, 8136, 7852, 8176, 8148] dup_rows_per_dst=[4, 14, 56, 862, 0, 4, 482, 14] combine_rows_per_src=[8180, 8184, 7232, 8192, 8136, 7852, 8176, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=220217344 dispatch_meta_bytes=218792 combine_payload_bytes=220217344 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=220436136 ingress_bytes=270548992 link_bytes=270548992 token_bytes=440653480 weight_movement_bytes=50331648 total_nvlink_bytes=490985128 weight_to_token_ratio=0.114220 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.058565227e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234848256 dispatch_meta_bytes=229376 combine_payload_bytes=234848256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=235077632 ingress_bytes=260014080 link_bytes=260014080 token_bytes=469925888 weight_movement_bytes=25165824 total_nvlink_bytes=495091712 weight_to_token_ratio=0.053553 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978406719e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=234700800 dispatch_meta_bytes=229728 combine_payload_bytes=234700800 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234930528 ingress_bytes=259866624 link_bytes=259866624 token_bytes=469631328 weight_movement_bytes=25165824 total_nvlink_bytes=494797152 weight_to_token_ratio=0.053586 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.977284749e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229728256 dispatch_meta_bytes=228208 combine_payload_bytes=229728256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229956464 ingress_bytes=254894080 link_bytes=254894080 token_bytes=459684720 weight_movement_bytes=25165824 total_nvlink_bytes=484850544 weight_to_token_ratio=0.054746 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.939449435e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65310 payload_rows_remote=57118 dup_rows=226 dup_rows_remote=226 dedup_ratio=0.003448 router_tokens_per_expert=[0, 4096, 0, 0, 0, 4040, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 4039, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8080, 8192, 8192, 8078, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 226, 0, 0, 0, 0, 0] combine_rows_per_src=[8080, 8192, 8192, 8078, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229179392 dispatch_meta_bytes=229496 combine_payload_bytes=229179392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229408888 ingress_bytes=254345216 link_bytes=254345216 token_bytes=458588280 weight_movement_bytes=25165824 total_nvlink_bytes=483754104 weight_to_token_ratio=0.054877 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.935273214e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57234 payload_rows_remote=49044 dup_rows=8302 dup_rows_remote=3730 dedup_ratio=0.126678 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 18, 0, 0, 4094, 3686, 0, 0, 1, 0, 2281, 0, 0, 0, 5, 0, 157, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 36, 8190, 8068, 8174, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 4572, 2614, 994, 120, 0] combine_rows_per_src=[8192, 8192, 36, 8190, 8068, 8174, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46468 payload_rows_remote=38276 dup_rows=19068 dup_rows_remote=19068 dedup_ratio=0.290955 router_tokens_per_expert=[57, 0, 0, 0, 4096, 0, 4096, 3903, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4083, 4034, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 23, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 0, 3514, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 0, 0, 0, 0, 67, 0] payload_rows_per_dst=[8192, 8192, 8168, 50, 838, 7480, 5356, 8192] dup_rows_per_dst=[9490, 0, 8068, 0, 46, 1330, 0, 134] combine_rows_per_src=[8192, 8192, 8168, 50, 838, 7480, 5356, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=233955328 dispatch_meta_bytes=229376 combine_payload_bytes=233955328 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=234184704 ingress_bytes=334618624 link_bytes=334618624 token_bytes=468140032 weight_movement_bytes=100663296 total_nvlink_bytes=568803328 weight_to_token_ratio=0.215028 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.546061098e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46324 payload_rows_remote=38146 dup_rows=19212 dup_rows_remote=7310 dedup_ratio=0.293152 router_tokens_per_expert=[0, 0, 0, 0, 4039, 0, 4080, 3971, 0, 0, 4080, 16, 0, 11, 0, 0, 0, 131, 8, 0, 4081, 3424, 29, 0, 0, 0, 2489, 0, 0, 8, 0, 0, 411, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2261, 0, 1909, 7, 39, 1, 4, 14, 0, 15, 0, 0, 1689, 0, 10, 0, 7, 0, 10, 0, 10] payload_rows_per_dst=[8178, 8192, 8190, 4994, 850, 4548, 3410, 7962] dup_rows_per_dst=[11902, 22, 7156, 0, 0, 66, 36, 30] combine_rows_per_src=[8178, 8192, 8190, 4994, 850, 4548, 3410, 7962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=200884224 dispatch_meta_bytes=211096 combine_payload_bytes=200884224 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201095320 ingress_bytes=226050048 link_bytes=226050048 token_bytes=401979544 weight_movement_bytes=25165824 total_nvlink_bytes=427145368 weight_to_token_ratio=0.062605 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.719979679e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156778496 dispatch_meta_bytes=229376 combine_payload_bytes=156778496 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=157007872 ingress_bytes=181944320 link_bytes=181944320 token_bytes=313786368 weight_movement_bytes=25165824 total_nvlink_bytes=338952192 weight_to_token_ratio=0.080201 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.384386050e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57306 payload_rows_remote=49130 dup_rows=8230 dup_rows_remote=6640 dedup_ratio=0.125580 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3534, 0, 0, 0, 0, 3145, 0, 0, 0, 44, 0, 0, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 14, 0] payload_rows_per_dst=[8192, 8192, 0, 8192, 8176, 8170, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 6378, 1590, 182, 52, 28] combine_rows_per_src=[8192, 8192, 0, 8192, 8176, 8170, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56442 payload_rows_remote=48866 dup_rows=9094 dup_rows_remote=9014 dedup_ratio=0.138763 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4096, 3953, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 610, 0, 3, 0, 0, 0, 390, 3240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 23, 0] payload_rows_per_dst=[1132, 8192, 6774, 8192, 8192, 7576, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 1226, 7230, 80, 512, 46] combine_rows_per_src=[1132, 8192, 6774, 8192, 8192, 7576, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55940 payload_rows_remote=49400 dup_rows=9596 dup_rows_remote=5380 dedup_ratio=0.146423 router_tokens_per_expert=[0, 0, 0, 0, 4076, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4081, 3856, 0, 0, 0, 0, 2251, 0, 0, 0, 61, 0, 12, 1201, 0, 0, 0, 0, 0, 0, 0, 1, 0, 580, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 6540, 4930, 5638, 8156, 6102, 8192] dup_rows_per_dst=[2068, 0, 4216, 1342, 274, 1030, 666, 0] combine_rows_per_src=[8190, 8192, 6540, 4930, 5638, 8156, 6102, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56636 payload_rows_remote=48454 dup_rows=8900 dup_rows_remote=8900 dedup_ratio=0.135803 router_tokens_per_expert=[0, 0, 0, 0, 4070, 0, 4087, 4092, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 2, 0, 4093, 3594, 3, 0, 0, 0, 254, 0, 0, 3, 1, 23, 0, 219, 1, 0, 0, 0, 0, 0, 0, 0, 7, 412, 0, 3978, 1, 6, 0, 0, 3, 0, 2, 1, 0, 3819, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[0, 8170, 8192, 8192, 7578, 8148, 8174, 8182] dup_rows_per_dst=[0, 0, 2, 556, 50, 676, 7616, 0] combine_rows_per_src=[0, 8170, 8192, 8192, 7578, 8148, 8174, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=156246016 dispatch_meta_bytes=181824 combine_payload_bytes=156246016 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=156427840 ingress_bytes=256909312 link_bytes=256909312 token_bytes=312673856 weight_movement_bytes=100663296 total_nvlink_bytes=413337152 weight_to_token_ratio=0.321943 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.954783022e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=201236480 dispatch_meta_bytes=223080 combine_payload_bytes=201236480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201459560 ingress_bytes=226402304 link_bytes=226402304 token_bytes=402696040 weight_movement_bytes=25165824 total_nvlink_bytes=427861864 weight_to_token_ratio=0.062493 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20736 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.722659940e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=200155136 dispatch_meta_bytes=231520 combine_payload_bytes=200155136 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=200386656 ingress_bytes=250486784 link_bytes=250486784 token_bytes=400541792 weight_movement_bytes=50331648 total_nvlink_bytes=450873440 weight_to_token_ratio=0.125659 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.905915005e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=202342400 dispatch_meta_bytes=219120 combine_payload_bytes=202342400 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=202561520 ingress_bytes=277839872 link_bytes=277839872 token_bytes=404903920 weight_movement_bytes=75497472 total_nvlink_bytes=480401392 weight_to_token_ratio=0.186458 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=11648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.114040400e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=198467584 dispatch_meta_bytes=229416 combine_payload_bytes=198467584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198697000 ingress_bytes=223633408 link_bytes=223633408 token_bytes=397164584 weight_movement_bytes=25165824 total_nvlink_bytes=422330408 weight_to_token_ratio=0.063364 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.701591841e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57126 payload_rows_remote=48934 dup_rows=8410 dup_rows_remote=7686 dedup_ratio=0.128326 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 0, 263, 0, 0, 0, 2, 0, 2, 3146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[0, 8192, 8192, 8192, 8192, 7974, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 530, 6200, 952, 724, 4] combine_rows_per_src=[0, 8192, 8192, 8192, 8192, 7974, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58342 payload_rows_remote=50182 dup_rows=7194 dup_rows_remote=7192 dedup_ratio=0.109772 router_tokens_per_expert=[12, 31, 0, 3550, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4079, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4045, 0] payload_rows_per_dst=[8192, 1132, 8192, 8160, 8192, 8190, 8192, 8092] dup_rows_per_dst=[7186, 0, 0, 2, 0, 0, 6, 0] combine_rows_per_src=[8192, 1132, 8192, 8160, 8192, 8190, 8192, 8092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57340 payload_rows_remote=49148 dup_rows=8196 dup_rows_remote=374 dedup_ratio=0.125061 router_tokens_per_expert=[0, 0, 0, 183, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3987, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 0] dup_rows_per_dst=[366, 7822, 0, 0, 0, 0, 8, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=200433664 dispatch_meta_bytes=226480 combine_payload_bytes=200433664 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200660144 ingress_bytes=225599488 link_bytes=225599488 token_bytes=401093808 weight_movement_bytes=25165824 total_nvlink_bytes=426259632 weight_to_token_ratio=0.062743 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23552 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.716551438e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56762 payload_rows_remote=48598 dup_rows=8774 dup_rows_remote=704 dedup_ratio=0.133881 router_tokens_per_expert=[0, 10, 3, 4025, 0, 0, 0, 0, 12, 71, 0, 0, 0, 0, 0, 13, 0, 4067, 0, 4048, 144, 0, 18, 19, 4, 4010, 0, 9, 0, 0, 0, 0, 2, 0, 38, 0, 0, 0, 68, 0, 11, 0, 0, 0, 0, 3914, 22, 0, 4072, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4079, 0, 0, 0, 4092, 0] payload_rows_per_dst=[8164, 8192, 8180, 8038, 8136, 7874, 8178, 0] dup_rows_per_dst=[8070, 184, 418, 8, 74, 20, 0, 0] combine_rows_per_src=[8164, 8192, 8180, 8038, 8136, 7874, 8178, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=205545472 dispatch_meta_bytes=229496 combine_payload_bytes=205545472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=205774968 ingress_bytes=230711296 link_bytes=230711296 token_bytes=411320440 weight_movement_bytes=25165824 total_nvlink_bytes=436486264 weight_to_token_ratio=0.061183 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.755446390e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=201310208 dispatch_meta_bytes=198088 combine_payload_bytes=201310208 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201508296 ingress_bytes=226476032 link_bytes=226476032 token_bytes=402818504 weight_movement_bytes=25165824 total_nvlink_bytes=427984328 weight_to_token_ratio=0.062474 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=21760 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.723220925e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62606 payload_rows_remote=54414 dup_rows=2930 dup_rows_remote=2930 dedup_ratio=0.044708 router_tokens_per_expert=[0, 61, 0, 1229, 0, 0, 0, 0, 0, 3025, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 3877, 0] payload_rows_per_dst=[7842, 6050, 8192, 8192, 8192, 8190, 8192, 7756] dup_rows_per_dst=[2568, 0, 0, 0, 0, 0, 0, 362] combine_rows_per_src=[7842, 6050, 8192, 8192, 8192, 8190, 8192, 7756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57340 payload_rows_remote=49156 dup_rows=8196 dup_rows_remote=8196 dedup_ratio=0.125061 router_tokens_per_expert=[0, 28, 0, 481, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3613, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4069, 0] payload_rows_per_dst=[972, 8184, 8192, 7232, 8192, 8184, 8192, 8192] dup_rows_per_dst=[46, 0, 0, 0, 0, 0, 0, 8150] combine_rows_per_src=[972, 8184, 8192, 7232, 8192, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57846 payload_rows_remote=49654 dup_rows=7690 dup_rows_remote=7690 dedup_ratio=0.117340 router_tokens_per_expert=[12, 0, 0, 576, 0, 0, 0, 0, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4009, 0] payload_rows_per_dst=[8192, 7756, 8192, 8190, 8192, 8192, 8192, 940] dup_rows_per_dst=[1176, 5390, 0, 1124, 0, 0, 0, 0] combine_rows_per_src=[8192, 7756, 8192, 8190, 8192, 8192, 8192, 940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57244 payload_rows_remote=49102 dup_rows=8292 dup_rows_remote=520 dedup_ratio=0.126526 router_tokens_per_expert=[0, 20, 2, 561, 0, 2, 0, 0, 0, 3786, 2, 2, 1, 0, 0, 3, 0, 4084, 0, 4088, 9, 0, 68, 0, 0, 4092, 0, 3, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 8, 0, 4090, 9, 4, 4089, 0, 0, 0, 0, 0, 1, 3, 0, 0, 4071, 0, 0, 0, 3756, 0] payload_rows_per_dst=[1136, 7578, 7644, 8192, 8174, 8192, 8186, 8142] dup_rows_per_dst=[34, 10, 142, 18, 0, 310, 6, 7772] combine_rows_per_src=[1136, 7578, 7644, 8192, 8174, 8192, 8186, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=199057408 dispatch_meta_bytes=197208 combine_payload_bytes=199057408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199254616 ingress_bytes=224223232 link_bytes=224223232 token_bytes=398312024 weight_movement_bytes=25165824 total_nvlink_bytes=423477848 weight_to_token_ratio=0.063181 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=20352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.706079720e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=201342976 dispatch_meta_bytes=229408 combine_payload_bytes=201342976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201572384 ingress_bytes=226508800 link_bytes=226508800 token_bytes=402915360 weight_movement_bytes=25165824 total_nvlink_bytes=428081184 weight_to_token_ratio=0.062459 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.723470252e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=222879744 dispatch_meta_bytes=229376 combine_payload_bytes=222879744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=223109120 ingress_bytes=248045568 link_bytes=248045568 token_bytes=445988864 weight_movement_bytes=25165824 total_nvlink_bytes=471154688 weight_to_token_ratio=0.056427 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.887340172e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=203382784 dispatch_meta_bytes=229376 combine_payload_bytes=203382784 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=203612160 ingress_bytes=304046080 link_bytes=304046080 token_bytes=406994944 weight_movement_bytes=100663296 total_nvlink_bytes=507658240 weight_to_token_ratio=0.247333 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.313439363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=201121792 dispatch_meta_bytes=198488 combine_payload_bytes=201121792 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=201320280 ingress_bytes=301785088 link_bytes=301785088 token_bytes=402442072 weight_movement_bytes=100663296 total_nvlink_bytes=503105368 weight_to_token_ratio=0.250131 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.296235826e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57594 payload_rows_remote=49402 dup_rows=7942 dup_rows_remote=7942 dedup_ratio=0.121185 router_tokens_per_expert=[0, 100, 0, 1600, 0, 0, 0, 0, 0, 2523, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 3967, 0] payload_rows_per_dst=[3396, 5046, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 0, 0, 7938] combine_rows_per_src=[3396, 5046, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65310 payload_rows_remote=57162 dup_rows=226 dup_rows_remote=224 dedup_ratio=0.003448 router_tokens_per_expert=[0, 0, 0, 30, 0, 0, 10, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 44, 0, 0, 4077, 0, 0, 48, 0, 0, 4096, 4095, 0, 0, 0, 0, 4070, 39, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8064, 8192, 8192, 8148, 8192, 8192, 8140, 8190] dup_rows_per_dst=[0, 0, 0, 2, 0, 146, 0, 78] combine_rows_per_src=[8064, 8192, 8192, 8148, 8192, 8192, 8140, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64732 payload_rows_remote=56550 dup_rows=804 dup_rows_remote=804 dedup_ratio=0.012268 router_tokens_per_expert=[0, 0, 0, 105, 0, 0, 1, 3618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 36, 24, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 515, 0, 0, 3861, 0, 0, 5, 0, 39, 4091, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7446, 8182, 8192, 8192, 8192, 8156, 8192, 8180] dup_rows_per_dst=[2, 0, 0, 118, 0, 606, 78, 0] combine_rows_per_src=[7446, 8182, 8192, 8192, 8192, 8156, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=202350592 dispatch_meta_bytes=229376 combine_payload_bytes=202350592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202579968 ingress_bytes=227516416 link_bytes=227516416 token_bytes=404930560 weight_movement_bytes=25165824 total_nvlink_bytes=430096384 weight_to_token_ratio=0.062148 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.731137045e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60966 payload_rows_remote=52782 dup_rows=4570 dup_rows_remote=1234 dedup_ratio=0.069733 router_tokens_per_expert=[14, 1, 0, 112, 0, 0, 175, 3949, 3, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 2, 409, 0, 4077, 5, 1, 4040, 0, 0, 0, 32, 9, 65, 4062, 1, 8, 10, 133, 11, 14, 0, 0, 0, 3162, 0, 1, 193, 2, 1, 4082, 4071, 0, 0, 0, 0, 4065, 0, 0, 0, 32, 0, 0, 0, 1] payload_rows_per_dst=[8184, 8192, 7122, 7122, 7700, 6928, 7578, 8140] dup_rows_per_dst=[3336, 28, 798, 30, 302, 52, 2, 22] combine_rows_per_src=[8184, 8192, 7122, 7122, 7700, 6928, 7578, 8140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234135552 dispatch_meta_bytes=229544 combine_payload_bytes=234135552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=234365096 ingress_bytes=284467200 link_bytes=284467200 token_bytes=468500648 weight_movement_bytes=50331648 total_nvlink_bytes=518832296 weight_to_token_ratio=0.107431 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.164466708e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=231628800 dispatch_meta_bytes=229416 combine_payload_bytes=231628800 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=231858216 ingress_bytes=256794624 link_bytes=256794624 token_bytes=463487016 weight_movement_bytes=25165824 total_nvlink_bytes=488652840 weight_to_token_ratio=0.054297 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.953910379e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64968 payload_rows_remote=56780 dup_rows=568 dup_rows_remote=544 dedup_ratio=0.008667 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 272, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4093, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7646, 8192, 8192, 8182, 8190, 8188, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 544, 0, 24, 0, 0] combine_rows_per_src=[7646, 8192, 8192, 8182, 8190, 8188, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65192 payload_rows_remote=57000 dup_rows=344 dup_rows_remote=314 dedup_ratio=0.005249 router_tokens_per_expert=[0, 0, 0, 48, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 122, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 34, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7850, 8192, 8192, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 244, 30, 68, 0, 0] combine_rows_per_src=[7850, 8192, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64018 payload_rows_remote=55826 dup_rows=1518 dup_rows_remote=1484 dedup_ratio=0.023163 router_tokens_per_expert=[0, 0, 0, 364, 0, 0, 3, 2976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 4091, 94, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 349, 0, 0, 4066, 0, 0, 0, 0, 301, 4096, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6686, 8192, 8192, 8186, 8192, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 34, 188, 72, 638, 586, 0] combine_rows_per_src=[6686, 8192, 8192, 8186, 8192, 8192, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=216195072 dispatch_meta_bytes=216064 combine_payload_bytes=216195072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=216411136 ingress_bytes=241360896 link_bytes=241360896 token_bytes=432606208 weight_movement_bytes=25165824 total_nvlink_bytes=457772032 weight_to_token_ratio=0.058173 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.836477541e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65218 payload_rows_remote=57056 dup_rows=318 dup_rows_remote=318 dedup_ratio=0.004852 router_tokens_per_expert=[3, 6, 0, 2829, 0, 0, 84, 1228, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 44, 0, 4093, 0, 2, 4093, 0, 0, 0, 1, 2, 0, 4057, 0, 2, 0, 69, 3, 3, 3, 0, 5, 3970, 0, 0, 0, 0, 4, 4092, 4081, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 8174, 8188, 8192, 8184, 7956, 8178, 8162] dup_rows_per_dst=[116, 0, 88, 4, 84, 0, 26, 0] combine_rows_per_src=[8184, 8174, 8188, 8192, 8184, 7956, 8178, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=232570880 dispatch_meta_bytes=229296 combine_payload_bytes=232570880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=232800176 ingress_bytes=257736704 link_bytes=257736704 token_bytes=465371056 weight_movement_bytes=25165824 total_nvlink_bytes=490536880 weight_to_token_ratio=0.054077 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.961078519e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233472000 dispatch_meta_bytes=229256 combine_payload_bytes=233472000 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=233701256 ingress_bytes=283803648 link_bytes=283803648 token_bytes=467173256 weight_movement_bytes=50331648 total_nvlink_bytes=517504904 weight_to_token_ratio=0.107737 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.159417844e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=228663296 dispatch_meta_bytes=229240 combine_payload_bytes=228663296 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=228892536 ingress_bytes=278994944 link_bytes=278994944 token_bytes=457555832 weight_movement_bytes=50331648 total_nvlink_bytes=507887480 weight_to_token_ratio=0.110001 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.122829163e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233701376 dispatch_meta_bytes=229496 combine_payload_bytes=233701376 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233930872 ingress_bytes=258867200 link_bytes=258867200 token_bytes=467632248 weight_movement_bytes=25165824 total_nvlink_bytes=492798072 weight_to_token_ratio=0.053815 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.969680287e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65440 payload_rows_remote=57248 dup_rows=96 dup_rows_remote=62 dedup_ratio=0.001465 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 4095, 10, 0, 4096, 0, 4047, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8114, 8192, 8192, 8174, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 60, 34, 0, 0, 0, 0] combine_rows_per_src=[8192, 8114, 8192, 8192, 8174, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65356 payload_rows_remote=57164 dup_rows=180 dup_rows_remote=180 dedup_ratio=0.002747 router_tokens_per_expert=[0, 0, 0, 1343, 0, 0, 74, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 7, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8022, 8192, 8192, 8192, 8192, 8184, 8192, 8190] dup_rows_per_dst=[0, 0, 152, 0, 14, 14, 0, 0] combine_rows_per_src=[8022, 8192, 8192, 8192, 8192, 8184, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65270 payload_rows_remote=57322 dup_rows=266 dup_rows_remote=266 dedup_ratio=0.004059 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 4096, 0, 3973, 0, 1, 0, 4073, 0, 120, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 4, 4090, 1, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 10, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7948, 8188, 8192, 8192, 8192, 8186, 8180] dup_rows_per_dst=[4, 0, 198, 14, 34, 16, 0, 0] combine_rows_per_src=[8192, 7948, 8188, 8192, 8192, 8192, 8186, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59902 payload_rows_remote=51714 dup_rows=5634 dup_rows_remote=5576 dedup_ratio=0.085968 router_tokens_per_expert=[0, 0, 0, 1, 16, 0, 19, 12, 3929, 0, 0, 4075, 0, 2103, 0, 0, 0, 4078, 0, 136, 9, 1, 1, 0, 0, 0, 0, 0, 0, 13, 0, 1964, 4078, 0, 0, 23, 44, 4083, 0, 0, 0, 0, 1, 0, 0, 4081, 3, 14, 0, 0, 1, 0, 0, 0, 4022, 0, 15, 21, 0, 9, 0, 0, 16, 0] payload_rows_per_dst=[8188, 7720, 7212, 4288, 8192, 8146, 8046, 8110] dup_rows_per_dst=[58, 4792, 66, 436, 198, 6, 0, 78] combine_rows_per_src=[8188, 7720, 7212, 4288, 8192, 8146, 8046, 8110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234487808 dispatch_meta_bytes=229240 combine_payload_bytes=234487808 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234717048 ingress_bytes=259653632 link_bytes=259653632 token_bytes=469204856 weight_movement_bytes=25165824 total_nvlink_bytes=494370680 weight_to_token_ratio=0.053635 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975664126e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234143744 dispatch_meta_bytes=229376 combine_payload_bytes=234143744 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=234373120 ingress_bytes=334807040 link_bytes=334807040 token_bytes=468516864 weight_movement_bytes=100663296 total_nvlink_bytes=569180160 weight_to_token_ratio=0.214855 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.547494726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234790912 dispatch_meta_bytes=230352 combine_payload_bytes=234790912 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=235021264 ingress_bytes=310288384 link_bytes=310288384 token_bytes=469812176 weight_movement_bytes=75497472 total_nvlink_bytes=545309648 weight_to_token_ratio=0.160697 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=28672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.360936084e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65072 payload_rows_remote=56880 dup_rows=464 dup_rows_remote=464 dedup_ratio=0.007080 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 4095, 0, 3892, 0, 0, 0, 4025, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7784, 8186, 8184, 8152, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 458, 2, 2, 0, 0, 0] combine_rows_per_src=[8190, 7784, 8186, 8184, 8152, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62368 payload_rows_remote=54322 dup_rows=3168 dup_rows_remote=3120 dedup_ratio=0.048340 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 4096, 0, 2586, 0, 0, 0, 3918, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1558, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5178, 8046, 8192, 8192, 8192, 8184, 8192] dup_rows_per_dst=[0, 0, 48, 3112, 6, 2, 0, 0] combine_rows_per_src=[8192, 5178, 8046, 8192, 8192, 8192, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65364 payload_rows_remote=57174 dup_rows=172 dup_rows_remote=172 dedup_ratio=0.002625 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4013, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8026, 8188, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 170, 0, 0, 0, 0] combine_rows_per_src=[8192, 8026, 8188, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=211820544 dispatch_meta_bytes=229160 combine_payload_bytes=211820544 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=212049704 ingress_bytes=236986368 link_bytes=236986368 token_bytes=423870248 weight_movement_bytes=25165824 total_nvlink_bytes=449036072 weight_to_token_ratio=0.059372 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.803192438e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65230 payload_rows_remote=57038 dup_rows=306 dup_rows_remote=290 dedup_ratio=0.004669 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 4075, 5, 1, 4087, 0, 4073, 0, 0, 0, 4086, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 0, 3, 0, 36, 4091, 0, 0, 3, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 4093, 6, 4, 0, 0, 4, 0, 0, 0, 4086, 0, 3, 0, 6, 2, 2, 0, 0, 0] payload_rows_per_dst=[8134, 8156, 8176, 8192, 8008, 8192, 8180, 8192] dup_rows_per_dst=[0, 46, 8, 38, 4, 22, 172, 16] combine_rows_per_src=[8134, 8156, 8176, 8192, 8008, 8192, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232980480 dispatch_meta_bytes=229376 combine_payload_bytes=232980480 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=233209856 ingress_bytes=283312128 link_bytes=283312128 token_bytes=466190336 weight_movement_bytes=50331648 total_nvlink_bytes=516521984 weight_to_token_ratio=0.107964 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.155677945e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=222502912 dispatch_meta_bytes=229768 combine_payload_bytes=222502912 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=222732680 ingress_bytes=272834560 link_bytes=272834560 token_bytes=445235592 weight_movement_bytes=50331648 total_nvlink_bytes=495567240 weight_to_token_ratio=0.113045 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.075955759e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234184704 dispatch_meta_bytes=229384 combine_payload_bytes=234184704 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234414088 ingress_bytes=309682176 link_bytes=309682176 token_bytes=468598792 weight_movement_bytes=75497472 total_nvlink_bytes=544096264 weight_to_token_ratio=0.161113 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.356323541e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65442 payload_rows_remote=57250 dup_rows=94 dup_rows_remote=94 dedup_ratio=0.001434 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4002, 4090, 0, 57, 0, 0, 0, 0, 19, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8174, 8192, 8190, 8118, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 92, 0] combine_rows_per_src=[8192, 8192, 8174, 8192, 8190, 8118, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233627648 dispatch_meta_bytes=229312 combine_payload_bytes=233627648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233856960 ingress_bytes=258793472 link_bytes=258793472 token_bytes=467484608 weight_movement_bytes=25165824 total_nvlink_bytes=492650432 weight_to_token_ratio=0.053832 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.969119303e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65114 payload_rows_remote=56922 dup_rows=422 dup_rows_remote=422 dedup_ratio=0.006439 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 3885, 0, 0, 0, 4096, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7770, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 316, 106, 0, 0, 0, 0] combine_rows_per_src=[8192, 7770, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65126 payload_rows_remote=56934 dup_rows=410 dup_rows_remote=410 dedup_ratio=0.006256 router_tokens_per_expert=[0, 4092, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 1, 0, 3884, 3812, 0, 415, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 8164, 8192, 7814, 8190] dup_rows_per_dst=[0, 0, 0, 0, 0, 408, 2, 0] combine_rows_per_src=[8190, 8192, 8192, 8192, 8164, 8192, 7814, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64526 payload_rows_remote=56334 dup_rows=1010 dup_rows_remote=828 dedup_ratio=0.015411 router_tokens_per_expert=[0, 4075, 36, 0, 42, 0, 9, 18, 4073, 0, 1, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 2, 2, 4009, 0, 2, 11, 13, 0, 0, 4, 0, 0, 1, 16, 0, 4047, 1, 0, 23, 0, 0, 0, 3748, 4008, 11, 284, 0, 8, 0, 0, 0, 1, 59, 0, 0, 9, 0, 108, 0, 0, 0, 0, 4078, 0] payload_rows_per_dst=[8192, 8148, 8026, 8156, 8192, 8036, 8140, 7636] dup_rows_per_dst=[182, 48, 0, 34, 720, 12, 14, 0] combine_rows_per_src=[8192, 8148, 8026, 8156, 8192, 8036, 8140, 7636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234496000 dispatch_meta_bytes=229376 combine_payload_bytes=234496000 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234725376 ingress_bytes=259661824 link_bytes=259661824 token_bytes=469221376 weight_movement_bytes=25165824 total_nvlink_bytes=494387200 weight_to_token_ratio=0.053633 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975726458e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233152512 dispatch_meta_bytes=229376 combine_payload_bytes=233152512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233381888 ingress_bytes=258318336 link_bytes=258318336 token_bytes=466534400 weight_movement_bytes=25165824 total_nvlink_bytes=491700224 weight_to_token_ratio=0.053942 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.965504067e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=233201664 dispatch_meta_bytes=229376 combine_payload_bytes=233201664 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=233431040 ingress_bytes=283533312 link_bytes=283533312 token_bytes=466632704 weight_movement_bytes=50331648 total_nvlink_bytes=516964352 weight_to_token_ratio=0.107861 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.157360900e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65036 payload_rows_remote=56844 dup_rows=500 dup_rows_remote=194 dedup_ratio=0.007629 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 153, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4096, 0] payload_rows_per_dst=[8192, 8192, 8188, 8192, 7696, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 306, 6, 188] combine_rows_per_src=[8192, 8192, 8188, 8192, 7696, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=16 dedup_ratio=0.000305 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 8, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8176, 8192] dup_rows_per_dst=[0, 0, 4, 0, 0, 16, 0, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8176, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65424 payload_rows_remote=57344 dup_rows=112 dup_rows_remote=112 dedup_ratio=0.001709 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8080, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2, 0, 0, 6, 0, 104] combine_rows_per_src=[8192, 8192, 8192, 8192, 8080, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6912593880 prefetch_bytes=377487360 reduce_bytes=402653184 weight_movement_bytes=780140544 egress_bytes=3458117592 ingress_bytes=4234616832 link_bytes=4234616832 total_nvlink_bytes=7692734424 weight_to_token_ratio=0.112858 prefetch_share_of_ingress=0.089143 dedup_ratio=0.075535 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=16 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039292 gemm_inner_pad_upper_bound=16256 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.013783829e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64714 payload_rows_remote=56532 dup_rows=822 dup_rows_remote=584 dedup_ratio=0.012543 router_tokens_per_expert=[0, 4092, 14, 0, 0, 0, 0, 0, 4091, 0, 5, 0, 2, 0, 4087, 0, 0, 0, 0, 0, 6, 0, 4082, 0, 0, 0, 2, 0, 0, 4, 0, 0, 4, 0, 0, 4062, 6, 3, 3, 0, 0, 0, 4093, 4069, 2, 1, 0, 5, 0, 0, 36, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4091, 0] payload_rows_per_dst=[7990, 8184, 8176, 8190, 8142, 7674, 8176, 8182] dup_rows_per_dst=[26, 4, 196, 4, 14, 10, 330, 238] combine_rows_per_src=[7990, 8184, 8176, 8190, 8142, 7674, 8176, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=230744064 dispatch_meta_bytes=228648 combine_payload_bytes=230744064 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=230972712 ingress_bytes=281075712 link_bytes=281075712 token_bytes=461716776 weight_movement_bytes=50331648 total_nvlink_bytes=512048424 weight_to_token_ratio=0.109010 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.138661403e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6722087096 prefetch_bytes=352321536 reduce_bytes=276824064 weight_movement_bytes=629145600 egress_bytes=3362875576 ingress_bytes=3988357120 link_bytes=3988357120 total_nvlink_bytes=7351232696 weight_to_token_ratio=0.093594 prefetch_share_of_ingress=0.088338 dedup_ratio=0.104630 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=11 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=14732 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.896674337e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=232833024 dispatch_meta_bytes=228152 combine_payload_bytes=232833024 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=233061176 ingress_bytes=358662144 link_bytes=358662144 token_bytes=465894200 weight_movement_bytes=125829120 total_nvlink_bytes=591723320 weight_to_token_ratio=0.270081 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.729004505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234799104 dispatch_meta_bytes=229360 combine_payload_bytes=234799104 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=235028464 ingress_bytes=259964928 link_bytes=259964928 token_bytes=469827568 weight_movement_bytes=25165824 total_nvlink_bytes=494993392 weight_to_token_ratio=0.053564 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978032729e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234881024 dispatch_meta_bytes=229824 combine_payload_bytes=234881024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=235110848 ingress_bytes=260046848 link_bytes=260046848 token_bytes=469991872 weight_movement_bytes=25165824 total_nvlink_bytes=495157696 weight_to_token_ratio=0.053545 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978656046e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=231555072 dispatch_meta_bytes=228464 combine_payload_bytes=231555072 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=231783536 ingress_bytes=281886720 link_bytes=281886720 token_bytes=463338608 weight_movement_bytes=50331648 total_nvlink_bytes=513670256 weight_to_token_ratio=0.108628 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.144832237e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6241460488 prefetch_bytes=402653184 reduce_bytes=377487360 weight_movement_bytes=780140544 egress_bytes=3122430216 ingress_bytes=3899170816 link_bytes=3899170816 total_nvlink_bytes=7021601032 weight_to_token_ratio=0.124993 prefetch_share_of_ingress=0.103266 dedup_ratio=0.151201 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=15 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=16129 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.854261542e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57288 dup_rows=64 dup_rows_remote=38 dedup_ratio=0.000977 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4091, 4092, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8146, 8182, 8184, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 26, 38] combine_rows_per_src=[8192, 8192, 8192, 8192, 8146, 8182, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6742202448 prefetch_bytes=377487360 reduce_bytes=578813952 weight_movement_bytes=956301312 egress_bytes=3372922960 ingress_bytes=4325580800 link_bytes=4325580800 total_nvlink_bytes=7698503760 weight_to_token_ratio=0.141838 prefetch_share_of_ingress=0.087269 dedup_ratio=0.094547 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=23 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039982 gemm_inner_pad_upper_bound=15748 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.057041997e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6825359568 prefetch_bytes=352321536 reduce_bytes=452984832 weight_movement_bytes=805306368 egress_bytes=3414522064 ingress_bytes=4216143872 link_bytes=4216143872 total_nvlink_bytes=7630665936 weight_to_token_ratio=0.117987 prefetch_share_of_ingress=0.083565 dedup_ratio=0.092247 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=18 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042509 gemm_inner_pad_upper_bound=15113 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.004998961e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6989435728 prefetch_bytes=402653184 reduce_bytes=402653184 weight_movement_bytes=805306368 egress_bytes=3496555344 ingress_bytes=4298186752 link_bytes=4298186752 total_nvlink_bytes=7794742096 weight_to_token_ratio=0.115218 prefetch_share_of_ingress=0.093680 dedup_ratio=0.073959 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=16 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=15748 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.044014681e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6406543408 prefetch_bytes=427819008 reduce_bytes=176160768 weight_movement_bytes=603979776 egress_bytes=3205052464 ingress_bytes=3805470720 link_bytes=3805470720 total_nvlink_bytes=7010523184 weight_to_token_ratio=0.094275 prefetch_share_of_ingress=0.112422 dedup_ratio=0.133871 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=7 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=15113 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.809702200e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234651648 dispatch_meta_bytes=229304 combine_payload_bytes=234651648 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=234880952 ingress_bytes=284983296 link_bytes=284983296 token_bytes=469532600 weight_movement_bytes=50331648 total_nvlink_bytes=519864248 weight_to_token_ratio=0.107195 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.168393603e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6759377056 prefetch_bytes=402653184 reduce_bytes=427819008 weight_movement_bytes=830472192 egress_bytes=3381520544 ingress_bytes=4208328704 link_bytes=4208328704 total_nvlink_bytes=7589849248 weight_to_token_ratio=0.122862 prefetch_share_of_ingress=0.095680 dedup_ratio=0.094822 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=17 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=15875 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.001282436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-09-21 08:31:46] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 220.3 | throughput per GPU (TFLOP/s/GPU): 146.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.817495E+00 | loss scale: 1.0 | grad norm: 8.746 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46142 payload_rows_remote=40150 dup_rows=19394 dup_rows_remote=16518 dedup_ratio=0.295929 router_tokens_per_expert=[92, 318, 825, 8, 0, 431, 887, 320, 2, 645, 32, 0, 5, 49, 453, 208, 1, 1354, 1057, 1427, 25, 109, 292, 4, 129, 2, 77, 338, 1110, 372, 1382, 1657, 618, 506, 566, 1494, 189, 1340, 879, 773, 1137, 366, 1575, 43, 864, 700, 3, 59, 823, 666, 793, 6, 89, 1376, 1791, 25, 1279, 31, 230, 2, 15, 9, 899, 11] payload_rows_per_dst=[5786, 5894, 6210, 5992, 5820, 4990, 5322, 6128] dup_rows_per_dst=[3126, 2562, 2328, 2876, 2508, 1354, 2234, 2406] combine_rows_per_src=[5786, 5894, 6210, 5992, 5820, 4990, 5322, 6128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45850 payload_rows_remote=38806 dup_rows=19686 dup_rows_remote=14926 dedup_ratio=0.300385 router_tokens_per_expert=[123, 280, 915, 8, 0, 622, 903, 309, 14, 534, 109, 3, 25, 15, 498, 112, 1, 908, 1194, 1695, 47, 174, 403, 28, 83, 12, 211, 270, 1063, 690, 848, 1460, 509, 378, 484, 1155, 104, 1263, 553, 1054, 1726, 916, 1614, 89, 897, 592, 8, 60, 829, 607, 786, 28, 147, 1046, 1850, 23, 1539, 31, 221, 22, 17, 27, 609, 27] payload_rows_per_dst=[4856, 5368, 6308, 4840, 6106, 7044, 5044, 6284] dup_rows_per_dst=[1464, 2088, 2592, 1514, 2978, 4760, 1888, 2402] combine_rows_per_src=[4856, 5368, 6308, 4840, 6106, 7044, 5044, 6284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45858 payload_rows_remote=39742 dup_rows=19678 dup_rows_remote=17282 dedup_ratio=0.300262 router_tokens_per_expert=[266, 269, 739, 10, 2, 518, 789, 378, 6, 909, 19, 2, 23, 51, 508, 116, 1, 1022, 980, 1744, 82, 147, 279, 1, 280, 25, 25, 337, 992, 542, 983, 1644, 1084, 321, 333, 1161, 146, 1079, 677, 1039, 1021, 725, 1625, 45, 645, 607, 5, 66, 732, 983, 701, 87, 171, 881, 1826, 83, 1488, 28, 277, 2, 10, 38, 1173, 20] payload_rows_per_dst=[5860, 5922, 6116, 6402, 4744, 4954, 5180, 6680] dup_rows_per_dst=[3332, 3058, 2396, 3254, 1224, 1274, 2096, 3044] combine_rows_per_src=[5860, 5922, 6116, 6402, 4744, 4954, 5180, 6680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45934 payload_rows_remote=39180 dup_rows=19602 dup_rows_remote=15592 dedup_ratio=0.299103 router_tokens_per_expert=[181, 336, 1008, 2, 0, 744, 945, 236, 1, 624, 31, 0, 8, 36, 256, 99, 10, 1514, 673, 1899, 59, 211, 230, 4, 147, 14, 59, 332, 945, 702, 766, 1562, 957, 504, 443, 1306, 148, 1208, 702, 1066, 1518, 603, 1712, 40, 854, 817, 4, 38, 1107, 566, 524, 30, 158, 1276, 1742, 77, 1135, 9, 208, 8, 6, 15, 328, 25] payload_rows_per_dst=[5354, 5204, 6358, 4668, 6754, 6702, 5366, 5528] dup_rows_per_dst=[2218, 1934, 2842, 1262, 4010, 3802, 2110, 1424] combine_rows_per_src=[5354, 5204, 6358, 4668, 6754, 6702, 5366, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45578 payload_rows_remote=39514 dup_rows=19958 dup_rows_remote=17350 dedup_ratio=0.304535 router_tokens_per_expert=[171, 243, 1138, 0, 0, 662, 1208, 400, 4, 604, 13, 5, 6, 56, 369, 145, 6, 1016, 888, 1589, 36, 166, 233, 10, 77, 11, 104, 278, 1308, 820, 1027, 1302, 703, 378, 430, 1069, 142, 1223, 475, 842, 1724, 484, 1714, 46, 808, 605, 24, 59, 976, 659, 643, 47, 117, 1448, 1900, 39, 1513, 35, 251, 10, 54, 11, 433, 11] payload_rows_per_dst=[6506, 6064, 5900, 6322, 3616, 5492, 5436, 6242] dup_rows_per_dst=[4566, 2608, 1988, 3532, 640, 2008, 2422, 2194] combine_rows_per_src=[6506, 6064, 5900, 6322, 3616, 5492, 5436, 6242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45554 payload_rows_remote=40144 dup_rows=19982 dup_rows_remote=18198 dedup_ratio=0.304901 router_tokens_per_expert=[226, 459, 690, 1, 0, 413, 964, 212, 12, 400, 13, 3, 28, 3, 443, 65, 18, 1337, 679, 1381, 89, 350, 519, 5, 121, 15, 73, 302, 1017, 703, 1008, 1522, 867, 594, 312, 862, 130, 1164, 603, 1145, 1434, 1015, 1999, 401, 552, 308, 46, 130, 1099, 625, 623, 86, 282, 1754, 1003, 69, 1498, 61, 306, 8, 29, 70, 594, 28] payload_rows_per_dst=[4698, 4568, 5380, 5692, 6670, 6258, 6878, 5410] dup_rows_per_dst=[1232, 1418, 1748, 2414, 3676, 2644, 5066, 1784] combine_rows_per_src=[4698, 4568, 5380, 5692, 6670, 6258, 6878, 5410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45608 payload_rows_remote=39434 dup_rows=19928 dup_rows_remote=16388 dedup_ratio=0.304077 router_tokens_per_expert=[232, 256, 729, 4, 3, 396, 920, 270, 3, 574, 33, 0, 19, 23, 557, 48, 7, 663, 539, 1687, 29, 307, 278, 14, 131, 14, 155, 307, 1172, 614, 882, 1547, 571, 639, 573, 1435, 234, 1984, 678, 1209, 1267, 566, 2047, 45, 798, 653, 3, 86, 962, 803, 449, 58, 144, 727, 1714, 37, 1616, 7, 293, 4, 92, 5, 642, 14] payload_rows_per_dst=[6174, 6862, 5546, 6370, 4382, 5166, 4820, 6288] dup_rows_per_dst=[3540, 4908, 1630, 3274, 1008, 1542, 1582, 2444] combine_rows_per_src=[6174, 6862, 5546, 6370, 4382, 5166, 4820, 6288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 3, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164454400 dispatch_meta_bytes=226672 combine_payload_bytes=164454400 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=164681072 ingress_bytes=214786048 link_bytes=214786048 token_bytes=329135472 weight_movement_bytes=50331648 total_nvlink_bytes=379467120 weight_to_token_ratio=0.152921 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.634273654e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158949376 dispatch_meta_bytes=214928 combine_payload_bytes=158949376 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159164304 ingress_bytes=234446848 link_bytes=234446848 token_bytes=318113680 weight_movement_bytes=75497472 total_nvlink_bytes=393611152 weight_to_token_ratio=0.237329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783869625e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162783232 dispatch_meta_bytes=228096 combine_payload_bytes=162783232 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=163011328 ingress_bytes=213114880 link_bytes=213114880 token_bytes=325794560 weight_movement_bytes=50331648 total_nvlink_bytes=376126208 weight_to_token_ratio=0.154489 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.621557996e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160481280 dispatch_meta_bytes=219088 combine_payload_bytes=160481280 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=160700368 ingress_bytes=261144576 link_bytes=261144576 token_bytes=321181648 weight_movement_bytes=100663296 total_nvlink_bytes=421844944 weight_to_token_ratio=0.313415 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=7296 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.987008487e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161849344 dispatch_meta_bytes=227456 combine_payload_bytes=161849344 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=162076800 ingress_bytes=237346816 link_bytes=237346816 token_bytes=323926144 weight_movement_bytes=75497472 total_nvlink_bytes=399423616 weight_to_token_ratio=0.233070 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=10496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.805935031e-04 prefetch_bytes_per_gemm_flop=5.744485294e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164429824 dispatch_meta_bytes=233368 combine_payload_bytes=164429824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=164663192 ingress_bytes=189595648 link_bytes=189595648 token_bytes=329093016 weight_movement_bytes=25165824 total_nvlink_bytes=354258840 weight_to_token_ratio=0.076470 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.442603816e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46582 payload_rows_remote=40308 dup_rows=18954 dup_rows_remote=15136 dedup_ratio=0.289215 router_tokens_per_expert=[141, 249, 1344, 0, 0, 541, 906, 231, 3, 859, 36, 0, 8, 3, 307, 86, 15, 803, 1000, 1549, 18, 324, 363, 25, 96, 18, 95, 445, 1126, 786, 996, 2071, 801, 371, 533, 1037, 115, 1200, 879, 930, 1063, 540, 1945, 110, 603, 394, 17, 25, 1087, 536, 588, 4, 114, 1765, 1611, 28, 1116, 20, 203, 3, 10, 5, 660, 11] payload_rows_per_dst=[5084, 5296, 6144, 5332, 7126, 5696, 6274, 5630] dup_rows_per_dst=[1740, 1782, 2050, 1792, 4274, 1850, 3818, 1648] combine_rows_per_src=[5084, 5296, 6144, 5332, 7126, 5696, 6274, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161521664 dispatch_meta_bytes=223288 combine_payload_bytes=161521664 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=161744952 ingress_bytes=186687488 link_bytes=186687488 token_bytes=323266616 weight_movement_bytes=25165824 total_nvlink_bytes=348432440 weight_to_token_ratio=0.077849 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.420476078e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50304 payload_rows_remote=47454 dup_rows=15232 dup_rows_remote=14884 dedup_ratio=0.232422 router_tokens_per_expert=[96, 12, 0, 0, 0, 92, 0, 226, 0, 59, 3923, 250, 0, 1, 88, 2, 3, 0, 1466, 0, 1041, 3, 31, 1, 0, 1151, 1, 2, 0, 32, 411, 2, 0, 0, 0, 0, 3688, 4030, 1875, 5, 1968, 0, 1202, 0, 3, 1, 16, 0, 0, 0, 47, 1329, 747, 0, 1, 0, 209, 1, 17, 2034, 3100, 0, 3601, 1] payload_rows_per_dst=[8146, 7958, 7516, 2850, 4028, 8020, 7438, 4348] dup_rows_per_dst=[766, 688, 3774, 348, 228, 5562, 3690, 176] combine_rows_per_src=[8146, 7958, 7516, 2850, 4028, 8020, 7438, 4348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47224 payload_rows_remote=39736 dup_rows=18312 dup_rows_remote=15232 dedup_ratio=0.279419 router_tokens_per_expert=[50, 0, 2, 0, 4, 655, 0, 56, 12, 411, 3997, 454, 0, 0, 131, 0, 0, 0, 1112, 0, 397, 3, 23, 91, 0, 560, 0, 67, 0, 22, 2832, 100, 4, 0, 3, 0, 3882, 2743, 1377, 0, 1669, 0, 86, 0, 7, 0, 7, 4, 0, 0, 2835, 372, 999, 6, 0, 17, 1150, 1, 131, 29, 2956, 0, 3511, 0] payload_rows_per_dst=[4390, 8058, 2926, 6278, 3946, 7488, 8098, 6040] dup_rows_per_dst=[600, 1952, 1318, 884, 852, 3080, 8124, 1502] combine_rows_per_src=[4390, 8058, 2926, 6278, 3946, 7488, 8098, 6040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55922 payload_rows_remote=48466 dup_rows=9614 dup_rows_remote=9612 dedup_ratio=0.146698 router_tokens_per_expert=[435, 0, 0, 0, 0, 779, 0, 55, 0, 0, 3962, 256, 0, 1, 342, 0, 0, 0, 89, 0, 1017, 234, 30, 0, 1, 1237, 0, 0, 0, 297, 2869, 0, 0, 0, 1, 0, 3138, 4084, 3727, 1, 291, 0, 134, 0, 0, 0, 10, 0, 1, 0, 438, 859, 302, 0, 0, 0, 61, 1, 140, 2583, 2223, 0, 3170, 0] payload_rows_per_dst=[8174, 8006, 6152, 6948, 7456, 6528, 7296, 5362] dup_rows_per_dst=[2532, 1116, 1034, 1860, 2, 682, 2180, 208] combine_rows_per_src=[8174, 8006, 6152, 6948, 7456, 6528, 7296, 5362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51054 payload_rows_remote=43294 dup_rows=14482 dup_rows_remote=12470 dedup_ratio=0.220978 router_tokens_per_expert=[492, 0, 0, 0, 0, 1224, 1, 0, 0, 131, 3010, 172, 0, 0, 1573, 0, 0, 0, 277, 11, 460, 3, 66, 16, 0, 1521, 0, 1, 0, 1028, 2365, 23, 0, 0, 1, 0, 3115, 3937, 1585, 0, 1722, 0, 182, 0, 0, 0, 0, 0, 5, 0, 15, 2571, 164, 0, 126, 4, 105, 8, 674, 1280, 1427, 0, 3473, 0] payload_rows_per_dst=[8066, 7760, 4054, 7072, 7110, 7604, 5474, 3914] dup_rows_per_dst=[3242, 2012, 466, 2804, 2292, 3150, 296, 220] combine_rows_per_src=[8066, 7760, 4054, 7072, 7110, 7604, 5474, 3914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165101568 dispatch_meta_bytes=221776 combine_payload_bytes=165101568 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=165323344 ingress_bytes=215433216 link_bytes=215433216 token_bytes=330424912 weight_movement_bytes=50331648 total_nvlink_bytes=380756560 weight_to_token_ratio=0.152324 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.639197854e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52290 payload_rows_remote=46548 dup_rows=13246 dup_rows_remote=12112 dedup_ratio=0.202118 router_tokens_per_expert=[430, 0, 0, 1, 0, 1347, 0, 87, 0, 261, 3901, 509, 0, 0, 116, 20, 0, 3, 493, 87, 6, 122, 446, 0, 2, 1827, 1, 0, 214, 85, 1569, 0, 0, 3, 0, 0, 2696, 4039, 1455, 3, 1277, 0, 379, 3, 0, 0, 112, 3, 0, 6, 17, 2168, 1240, 0, 0, 0, 1092, 0, 41, 2340, 2281, 0, 2086, 0] payload_rows_per_dst=[8154, 7894, 5742, 5860, 6994, 5966, 5772, 5908] dup_rows_per_dst=[3654, 1720, 1134, 1536, 1320, 1754, 1090, 1038] combine_rows_per_src=[8154, 7894, 5742, 5860, 6994, 5966, 5772, 5908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45032 payload_rows_remote=37014 dup_rows=20504 dup_rows_remote=12050 dedup_ratio=0.312866 router_tokens_per_expert=[1444, 0, 149, 0, 0, 1331, 0, 159, 14, 999, 1384, 292, 0, 0, 4, 2, 0, 0, 3402, 2, 3383, 7, 23, 0, 0, 141, 0, 0, 2, 46, 2450, 6, 0, 0, 0, 0, 1185, 3838, 1088, 3, 147, 0, 396, 0, 0, 0, 103, 0, 1, 1, 510, 308, 516, 0, 6, 0, 163, 0, 1378, 561, 3750, 0, 3574, 0] payload_rows_per_dst=[4900, 2534, 7924, 6158, 7778, 3338, 4382, 8018] dup_rows_per_dst=[1266, 88, 5710, 1900, 2080, 334, 672, 8454] combine_rows_per_src=[4900, 2534, 7924, 6158, 7778, 3338, 4382, 8018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=194371584 dispatch_meta_bytes=249352 combine_payload_bytes=194371584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=194620936 ingress_bytes=219537408 link_bytes=219537408 token_bytes=388992520 weight_movement_bytes=25165824 total_nvlink_bytes=414158344 weight_to_token_ratio=0.064695 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.670426013e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=162758656 dispatch_meta_bytes=219872 combine_payload_bytes=162758656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=162978528 ingress_bytes=187924480 link_bytes=187924480 token_bytes=325737184 weight_movement_bytes=25165824 total_nvlink_bytes=350903008 weight_to_token_ratio=0.077258 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.429888158e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49250 payload_rows_remote=41578 dup_rows=16286 dup_rows_remote=14604 dedup_ratio=0.248505 router_tokens_per_expert=[4, 0, 13, 0, 0, 884, 0, 47, 0, 11, 4060, 287, 5, 0, 251, 21, 0, 0, 1378, 0, 1936, 18, 947, 0, 3, 2119, 1, 0, 155, 21, 347, 13, 1, 14, 0, 1, 2443, 3729, 1197, 0, 347, 8, 381, 1, 1, 0, 0, 0, 0, 0, 234, 730, 729, 1, 0, 0, 482, 0, 2145, 2322, 2959, 0, 2520, 2] payload_rows_per_dst=[7672, 8148, 7914, 4740, 6324, 5542, 3094, 5816] dup_rows_per_dst=[1682, 3656, 6562, 578, 1442, 974, 294, 1098] combine_rows_per_src=[7672, 8148, 7914, 4740, 6324, 5542, 3094, 5816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=198516736 dispatch_meta_bytes=232312 combine_payload_bytes=198516736 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=198749048 ingress_bytes=274014208 link_bytes=274014208 token_bytes=397265784 weight_movement_bytes=75497472 total_nvlink_bytes=472763256 weight_to_token_ratio=0.190043 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.084931517e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=177332224 dispatch_meta_bytes=223056 combine_payload_bytes=177332224 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=177555280 ingress_bytes=227663872 link_bytes=227663872 token_bytes=354887504 weight_movement_bytes=50331648 total_nvlink_bytes=405219152 weight_to_token_ratio=0.141824 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.732259015e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=190660608 dispatch_meta_bytes=234640 combine_payload_bytes=190660608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=190895248 ingress_bytes=215826432 link_bytes=215826432 token_bytes=381555856 weight_movement_bytes=25165824 total_nvlink_bytes=406721680 weight_to_token_ratio=0.065956 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.642189774e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=151609344 dispatch_meta_bytes=196256 combine_payload_bytes=151609344 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=151805600 ingress_bytes=252272640 link_bytes=252272640 token_bytes=303414944 weight_movement_bytes=100663296 total_nvlink_bytes=404078240 weight_to_token_ratio=0.331768 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=12160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.919503305e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=170303488 dispatch_meta_bytes=224728 combine_payload_bytes=170303488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=170528216 ingress_bytes=195469312 link_bytes=195469312 token_bytes=340831704 weight_movement_bytes=25165824 total_nvlink_bytes=365997528 weight_to_token_ratio=0.073837 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.487295612e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43760 payload_rows_remote=37316 dup_rows=21776 dup_rows_remote=20584 dedup_ratio=0.332275 router_tokens_per_expert=[517, 0, 0, 1, 0, 612, 3, 13, 8, 3, 3991, 5, 0, 0, 728, 2, 0, 0, 1325, 41, 162, 88, 34, 5, 0, 886, 0, 7, 10, 1307, 2879, 7, 2, 0, 0, 0, 2755, 3300, 3540, 0, 1812, 0, 131, 0, 0, 0, 137, 0, 1, 13, 151, 297, 592, 5, 1, 3, 9, 0, 614, 2499, 3595, 0, 677, 0] payload_rows_per_dst=[2216, 3588, 3120, 7978, 7720, 4780, 6444, 7914] dup_rows_per_dst=[76, 358, 190, 7742, 5964, 734, 1192, 5520] combine_rows_per_src=[2216, 3588, 3120, 7978, 7720, 4780, 6444, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52930 payload_rows_remote=44754 dup_rows=12606 dup_rows_remote=12496 dedup_ratio=0.192352 router_tokens_per_expert=[4, 3, 27, 0, 4037, 0, 1639, 0, 158, 0, 0, 2504, 1, 4092, 0, 1430, 3416, 0, 1433, 0, 0, 0, 0, 0, 0, 0, 1, 0, 59, 7, 0, 0, 7, 601, 10, 0, 1, 224, 7, 0, 1, 0, 2, 1, 2333, 0, 0, 0, 23, 1, 4076, 37, 2, 3257, 3299, 1, 1, 0, 0, 0, 72, 0, 1, 0] payload_rows_per_dst=[3308, 8038, 7394, 8176, 8186, 4672, 6544, 6612] dup_rows_per_dst=[38, 7940, 2696, 110, 1588, 2, 98, 134] combine_rows_per_src=[3308, 8038, 7394, 8176, 8186, 4672, 6544, 6612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57838 payload_rows_remote=50146 dup_rows=7698 dup_rows_remote=5112 dedup_ratio=0.117462 router_tokens_per_expert=[0, 1, 0, 0, 4063, 0, 4014, 6, 248, 0, 0, 2756, 0, 1425, 0, 1796, 1944, 0, 1511, 5, 0, 10, 0, 0, 0, 0, 1, 11, 1, 0, 0, 0, 50, 51, 0, 0, 18, 3, 82, 0, 0, 0, 0, 0, 3693, 21, 0, 0, 10, 0, 4092, 140, 0, 2460, 3914, 0, 0, 0, 0, 0, 320, 4, 118, 0] payload_rows_per_dst=[8040, 7036, 5776, 8186, 8136, 7692, 5068, 7904] dup_rows_per_dst=[2, 2564, 1164, 24, 398, 2586, 152, 808] combine_rows_per_src=[8040, 7036, 5776, 8186, 8136, 7692, 5068, 7904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56416 payload_rows_remote=49680 dup_rows=9120 dup_rows_remote=7526 dedup_ratio=0.139160 router_tokens_per_expert=[0, 0, 0, 0, 3190, 0, 3530, 0, 515, 0, 0, 1779, 0, 4059, 0, 42, 1702, 0, 505, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 2, 1580, 0, 0, 3, 72, 13, 0, 0, 0, 0, 6, 3556, 0, 0, 0, 1, 0, 4095, 25, 0, 3541, 4089, 0, 10, 0, 0, 0, 435, 0, 0, 0] payload_rows_per_dst=[7766, 6736, 4252, 8192, 7082, 7112, 7096, 8180] dup_rows_per_dst=[3754, 1594, 164, 32, 2638, 12, 38, 888] combine_rows_per_src=[7766, 6736, 4252, 8192, 7082, 7112, 7096, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53860 payload_rows_remote=47362 dup_rows=11676 dup_rows_remote=10878 dedup_ratio=0.178162 router_tokens_per_expert=[0, 0, 0, 0, 3189, 0, 3268, 0, 30, 0, 7, 1029, 1, 3132, 1, 870, 3397, 3, 1463, 8, 3, 0, 0, 0, 0, 0, 1, 52, 0, 0, 0, 0, 0, 424, 0, 0, 1, 1, 32, 1, 0, 1, 0, 6, 3262, 0, 3, 0, 5, 0, 4073, 1433, 0, 2280, 3984, 0, 0, 1, 0, 0, 677, 130, 0, 0] payload_rows_per_dst=[6536, 3524, 8124, 8160, 6498, 6540, 6312, 8166] dup_rows_per_dst=[0, 352, 7840, 92, 798, 52, 1124, 1418] combine_rows_per_src=[6536, 3524, 8124, 8160, 6498, 6540, 6312, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57472 payload_rows_remote=49548 dup_rows=8064 dup_rows_remote=4704 dedup_ratio=0.123047 router_tokens_per_expert=[0, 2, 0, 0, 4089, 2, 1204, 0, 16, 0, 0, 205, 0, 4080, 0, 226, 1980, 13, 3644, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 753, 0, 0, 133, 2, 0, 0, 0, 1, 3, 5, 3965, 1, 0, 0, 2, 35, 4050, 762, 13, 3219, 3804, 0, 0, 11, 0, 0, 537, 0, 4, 1] payload_rows_per_dst=[2412, 8160, 7924, 8102, 8178, 7934, 7124, 7638] dup_rows_per_dst=[4, 894, 3360, 0, 1776, 16, 938, 1076] combine_rows_per_src=[2412, 8160, 7924, 8102, 8178, 7934, 7124, 7638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52834 payload_rows_remote=44656 dup_rows=12702 dup_rows_remote=12110 dedup_ratio=0.193817 router_tokens_per_expert=[0, 0, 0, 0, 3137, 0, 3702, 0, 2852, 0, 0, 1512, 0, 4091, 0, 751, 91, 4, 1458, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1923, 52, 0, 0, 0, 125, 0, 0, 0, 0, 0, 863, 0, 0, 0, 0, 1, 4082, 427, 8, 3299, 4062, 0, 0, 44, 0, 0, 268, 3, 8, 0] payload_rows_per_dst=[7404, 6270, 3056, 8162, 7062, 5744, 6958, 8178] dup_rows_per_dst=[0, 3960, 56, 0, 3412, 476, 4206, 592] combine_rows_per_src=[7404, 6270, 3056, 8162, 7062, 5744, 6958, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=183312384 dispatch_meta_bytes=229000 combine_payload_bytes=183312384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183541384 ingress_bytes=208478208 link_bytes=208478208 token_bytes=366853768 weight_movement_bytes=25165824 total_nvlink_bytes=392019592 weight_to_token_ratio=0.068599 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.586278280e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=205398016 dispatch_meta_bytes=221032 combine_payload_bytes=205398016 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=205619048 ingress_bytes=230563840 link_bytes=230563840 token_bytes=411017064 weight_movement_bytes=25165824 total_nvlink_bytes=436182888 weight_to_token_ratio=0.061228 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.754324421e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=152846336 dispatch_meta_bytes=231600 combine_payload_bytes=152846336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=153077936 ingress_bytes=178012160 link_bytes=178012160 token_bytes=305924272 weight_movement_bytes=25165824 total_nvlink_bytes=331090096 weight_to_token_ratio=0.082262 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.354466856e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49234 payload_rows_remote=41042 dup_rows=16302 dup_rows_remote=6366 dedup_ratio=0.248749 router_tokens_per_expert=[0, 0, 45, 0, 4086, 1, 3820, 0, 1059, 2, 0, 612, 0, 3224, 0, 1472, 2493, 157, 326, 0, 18, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 1, 2, 327, 0, 4, 0, 0, 7, 0, 1, 0, 0, 1, 3886, 0, 0, 0, 0, 0, 4072, 108, 0, 3559, 3382, 0, 1, 52, 0, 3, 42, 2, 0, 0] payload_rows_per_dst=[8192, 4378, 5520, 8148, 4336, 7776, 7346, 3538] dup_rows_per_dst=[9936, 1912, 468, 4, 568, 0, 3372, 42] combine_rows_per_src=[8192, 4378, 5520, 8148, 4336, 7776, 7346, 3538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=203489280 dispatch_meta_bytes=228824 combine_payload_bytes=203489280 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=203718104 ingress_bytes=304152576 link_bytes=304152576 token_bytes=407207384 weight_movement_bytes=100663296 total_nvlink_bytes=507870680 weight_to_token_ratio=0.247204 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11136 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.314249674e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193994752 dispatch_meta_bytes=232960 combine_payload_bytes=193994752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=194227712 ingress_bytes=219160576 link_bytes=219160576 token_bytes=388222464 weight_movement_bytes=25165824 total_nvlink_bytes=413388288 weight_to_token_ratio=0.064823 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.667558757e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=202948608 dispatch_meta_bytes=217008 combine_payload_bytes=202948608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203165616 ingress_bytes=228114432 link_bytes=228114432 token_bytes=406114224 weight_movement_bytes=25165824 total_nvlink_bytes=431280048 weight_to_token_ratio=0.061967 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.735687256e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=182910976 dispatch_meta_bytes=227064 combine_payload_bytes=182910976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183138040 ingress_bytes=208076800 link_bytes=208076800 token_bytes=366049016 weight_movement_bytes=25165824 total_nvlink_bytes=391214840 weight_to_token_ratio=0.068750 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=28928 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.583224029e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=168108032 dispatch_meta_bytes=189632 combine_payload_bytes=168108032 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=168297664 ingress_bytes=218439680 link_bytes=218439680 token_bytes=336405696 weight_movement_bytes=50331648 total_nvlink_bytes=386737344 weight_to_token_ratio=0.149616 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25344 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.662073572e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59804 payload_rows_remote=51616 dup_rows=5732 dup_rows_remote=3390 dedup_ratio=0.087463 router_tokens_per_expert=[0, 0, 0, 1, 1, 4049, 0, 0, 0, 262, 10, 0, 0, 1784, 0, 3, 2, 0, 3812, 4039, 0, 0, 2, 0, 0, 2, 0, 0, 2, 1168, 0, 4093, 0, 8, 3829, 0, 1, 0, 0, 0, 8, 1, 3896, 1, 7, 0, 0, 0, 0, 1589, 0, 0, 4011, 1, 185, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8100, 3982, 7628, 8188, 7664, 8158, 8006, 8078] dup_rows_per_dst=[2, 136, 4, 2342, 12, 128, 3106, 2] combine_rows_per_src=[8100, 3982, 7628, 8188, 7664, 8158, 8006, 8078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59612 payload_rows_remote=51734 dup_rows=5924 dup_rows_remote=5924 dedup_ratio=0.090393 router_tokens_per_expert=[1, 0, 0, 2, 0, 4086, 1, 0, 1, 2414, 0, 0, 3, 16, 0, 0, 0, 0, 3720, 4082, 2, 24, 0, 0, 0, 0, 0, 18, 1, 1203, 0, 4042, 0, 0, 3567, 0, 5, 3, 0, 7, 0, 0, 3935, 0, 0, 0, 0, 4, 0, 2485, 5, 0, 3063, 1, 51, 0, 0, 0, 0, 0, 24, 2, 0, 0] payload_rows_per_dst=[8178, 4846, 7450, 8134, 7146, 7878, 7812, 8168] dup_rows_per_dst=[2, 22, 42, 2394, 18, 0, 3398, 48] combine_rows_per_src=[8178, 4846, 7450, 8134, 7146, 7878, 7812, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53888 payload_rows_remote=46816 dup_rows=11648 dup_rows_remote=9560 dedup_ratio=0.177734 router_tokens_per_expert=[0, 0, 0, 0, 1948, 0, 4072, 0, 314, 0, 0, 719, 0, 4025, 1, 55, 3762, 7, 76, 9, 17, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 307, 0, 3, 0, 160, 22, 0, 0, 0, 0, 0, 3435, 1, 0, 0, 0, 1, 4072, 3069, 0, 1510, 4015, 0, 0, 0, 0, 4, 1138, 14, 10, 0] payload_rows_per_dst=[8144, 2122, 7592, 8146, 4534, 8186, 7072, 8092] dup_rows_per_dst=[0, 56, 150, 2, 346, 6736, 2088, 2270] combine_rows_per_src=[8144, 2122, 7592, 8146, 4534, 8186, 7072, 8092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55360 payload_rows_remote=48482 dup_rows=10176 dup_rows_remote=9054 dedup_ratio=0.155273 router_tokens_per_expert=[0, 0, 0, 0, 0, 3926, 0, 0, 0, 2599, 0, 0, 0, 1401, 0, 0, 0, 0, 3788, 4039, 0, 0, 20, 0, 0, 0, 0, 8, 0, 840, 0, 4096, 0, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 520, 0, 0, 3717, 0, 3, 0, 0, 0, 0, 15, 89, 0, 0, 0] payload_rows_per_dst=[8076, 6878, 7658, 8192, 7774, 7640, 1046, 8096] dup_rows_per_dst=[5690, 1122, 1478, 1696, 0, 0, 0, 190] combine_rows_per_src=[8076, 6878, 7658, 8192, 7774, 7640, 1046, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=211419136 dispatch_meta_bytes=220024 combine_payload_bytes=211419136 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=211639160 ingress_bytes=286916608 link_bytes=286916608 token_bytes=423058296 weight_movement_bytes=75497472 total_nvlink_bytes=498555768 weight_to_token_ratio=0.178456 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.183103873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59170 payload_rows_remote=51672 dup_rows=6366 dup_rows_remote=6364 dedup_ratio=0.097137 router_tokens_per_expert=[0, 0, 1, 0, 1, 4093, 0, 0, 1, 17, 0, 0, 0, 1319, 0, 8, 0, 0, 4067, 4086, 0, 0, 0, 0, 0, 0, 0, 9, 0, 533, 0, 4092, 2, 1, 3747, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 2, 0, 2732, 0, 0, 3909, 0, 42, 0, 0, 0, 0, 0, 43, 0, 0, 1] payload_rows_per_dst=[8188, 2686, 8134, 8192, 7498, 8126, 8172, 8174] dup_rows_per_dst=[2, 4, 0, 1076, 2, 2, 5194, 86] combine_rows_per_src=[8188, 2686, 8134, 8192, 7498, 8126, 8172, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=211902464 dispatch_meta_bytes=230632 combine_payload_bytes=211902464 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=212133096 ingress_bytes=237068288 link_bytes=237068288 token_bytes=424035560 weight_movement_bytes=25165824 total_nvlink_bytes=449201384 weight_to_token_ratio=0.059348 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.803815754e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56160 payload_rows_remote=48510 dup_rows=9376 dup_rows_remote=5654 dedup_ratio=0.143066 router_tokens_per_expert=[0, 0, 0, 0, 0, 3551, 2, 0, 0, 2486, 0, 0, 0, 3439, 0, 33, 0, 4, 3422, 3985, 0, 0, 3, 0, 2, 0, 2, 0, 0, 0, 0, 4090, 5, 0, 3862, 0, 0, 0, 0, 0, 5, 0, 4094, 174, 1, 0, 0, 0, 0, 721, 0, 0, 2633, 3, 0, 0, 0, 0, 0, 0, 250, 0, 0, 1] payload_rows_per_dst=[7106, 7774, 7650, 8184, 7734, 8192, 1448, 8072] dup_rows_per_dst=[0, 4142, 3722, 4, 0, 1108, 0, 400] combine_rows_per_src=[7106, 7774, 7650, 8184, 7734, 8192, 1448, 8072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49364 payload_rows_remote=43676 dup_rows=16172 dup_rows_remote=14166 dedup_ratio=0.246765 router_tokens_per_expert=[0, 0, 0, 0, 0, 1687, 0, 0, 0, 3330, 0, 0, 0, 2761, 0, 11, 0, 0, 4096, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1003, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4090, 32, 0, 0, 0, 1, 0, 745, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0] payload_rows_per_dst=[3374, 7724, 5982, 2042, 8192, 8180, 8182, 5688] dup_rows_per_dst=[0, 4480, 0, 2, 7914, 66, 1704, 2006] combine_rows_per_src=[3374, 7724, 5982, 2042, 8192, 8180, 8182, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57356 payload_rows_remote=49204 dup_rows=8180 dup_rows_remote=5988 dedup_ratio=0.124817 router_tokens_per_expert=[1, 0, 13, 0, 2, 4062, 0, 0, 1, 1902, 4, 2, 0, 3202, 0, 71, 0, 0, 3903, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 8, 334, 6, 4035, 0, 0, 2727, 4, 0, 10, 2, 0, 53, 0, 4094, 6, 0, 0, 0, 0, 0, 2852, 0, 1, 1094, 0, 267, 0, 4, 0, 4, 0, 7, 0, 3, 0] payload_rows_per_dst=[8152, 7506, 7806, 6248, 5480, 8188, 5786, 8190] dup_rows_per_dst=[2192, 4808, 0, 570, 6, 118, 454, 32] combine_rows_per_src=[8152, 7506, 7806, 6248, 5480, 8188, 5786, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=191758336 dispatch_meta_bytes=225504 combine_payload_bytes=191758336 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=191983840 ingress_bytes=267255808 link_bytes=267255808 token_bytes=383742176 weight_movement_bytes=75497472 total_nvlink_bytes=459239648 weight_to_token_ratio=0.196740 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.033507902e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=198582272 dispatch_meta_bytes=230144 combine_payload_bytes=198582272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198812416 ingress_bytes=223748096 link_bytes=223748096 token_bytes=397394688 weight_movement_bytes=25165824 total_nvlink_bytes=422560512 weight_to_token_ratio=0.063327 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.702464484e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=211648512 dispatch_meta_bytes=232144 combine_payload_bytes=211648512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=211880656 ingress_bytes=236814336 link_bytes=236814336 token_bytes=423529168 weight_movement_bytes=25165824 total_nvlink_bytes=448694992 weight_to_token_ratio=0.059419 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.801883473e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=198696960 dispatch_meta_bytes=216656 combine_payload_bytes=198696960 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=198913616 ingress_bytes=274194432 link_bytes=274194432 token_bytes=397610576 weight_movement_bytes=75497472 total_nvlink_bytes=473108048 weight_to_token_ratio=0.189878 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.086302813e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=178896896 dispatch_meta_bytes=231368 combine_payload_bytes=178896896 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=179128264 ingress_bytes=229228544 link_bytes=229228544 token_bytes=358025160 weight_movement_bytes=50331648 total_nvlink_bytes=408356808 weight_to_token_ratio=0.140581 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.744164361e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=201539584 dispatch_meta_bytes=220768 combine_payload_bytes=201539584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201760352 ingress_bytes=226705408 link_bytes=226705408 token_bytes=403299936 weight_movement_bytes=25165824 total_nvlink_bytes=428465760 weight_to_token_ratio=0.062400 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.724966211e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62138 payload_rows_remote=55044 dup_rows=3398 dup_rows_remote=3398 dedup_ratio=0.051849 router_tokens_per_expert=[0, 2, 0, 0, 3476, 0, 0, 2780, 4096, 7, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 458, 0, 0, 0, 4, 0, 0, 0, 3543, 2, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4094, 4092, 0, 3917, 0, 0, 0, 0, 3, 0, 216, 16, 4041, 0, 189, 0, 0, 0, 1, 0, 0, 1822] payload_rows_per_dst=[6956, 8192, 8190, 7094, 8192, 7834, 8152, 7528] dup_rows_per_dst=[0, 20, 916, 0, 6, 0, 400, 2056] combine_rows_per_src=[6956, 8192, 8190, 7094, 8192, 7834, 8152, 7528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54394 payload_rows_remote=46222 dup_rows=11142 dup_rows_remote=10976 dedup_ratio=0.170013 router_tokens_per_expert=[0, 0, 0, 4, 92, 0, 2, 3385, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1, 2258, 0, 0, 0, 0, 134, 0, 0, 2817, 0, 0, 19, 0, 1, 0, 0, 4, 0, 11, 4020, 4090, 0, 4055, 103, 0, 14, 0, 1, 1, 235, 1, 3718, 3, 3386, 1, 0, 0, 0, 0, 1, 294] payload_rows_per_dst=[196, 8190, 8126, 5790, 8192, 8172, 7556, 8172] dup_rows_per_dst=[0, 0, 4476, 112, 36, 166, 390, 5962] combine_rows_per_src=[196, 8190, 8126, 5790, 8192, 8172, 7556, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59122 payload_rows_remote=51598 dup_rows=6414 dup_rows_remote=2710 dedup_ratio=0.097870 router_tokens_per_expert=[1, 0, 0, 0, 0, 3236, 0, 0, 0, 1747, 1, 0, 0, 2965, 0, 0, 0, 0, 3988, 4092, 3, 0, 0, 0, 0, 0, 1, 0, 1, 39, 0, 4095, 0, 0, 3037, 0, 11, 0, 0, 0, 0, 0, 3873, 3, 0, 0, 0, 0, 0, 3390, 0, 0, 2196, 0, 28, 0, 0, 0, 0, 0, 52, 9, 0, 0] payload_rows_per_dst=[6472, 6912, 7982, 8192, 6096, 7752, 7524, 8192] dup_rows_per_dst=[2, 2514, 0, 80, 0, 0, 3704, 114] combine_rows_per_src=[6472, 6912, 7982, 8192, 6096, 7752, 7524, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=225460224 dispatch_meta_bytes=233768 combine_payload_bytes=225460224 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=225693992 ingress_bytes=250626048 link_bytes=250626048 token_bytes=451154216 weight_movement_bytes=25165824 total_nvlink_bytes=476320040 weight_to_token_ratio=0.055781 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.906974643e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54984 payload_rows_remote=46792 dup_rows=10552 dup_rows_remote=10550 dedup_ratio=0.161011 router_tokens_per_expert=[0, 26, 0, 0, 819, 0, 0, 3971, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 38, 0, 0, 3516, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 4063, 2, 4090, 1380, 0, 0, 0, 0, 0, 1113, 20, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 1882] payload_rows_per_dst=[7994, 8192, 8188, 8192, 8130, 2762, 7762, 3764] dup_rows_per_dst=[1638, 2, 662, 7096, 0, 2, 1152, 0] combine_rows_per_src=[7994, 8192, 8188, 8192, 8130, 2762, 7762, 3764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56624 payload_rows_remote=48434 dup_rows=8912 dup_rows_remote=8908 dedup_ratio=0.135986 router_tokens_per_expert=[0, 2, 0, 0, 2435, 3, 0, 3840, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 817, 1, 0, 0, 0, 0, 0, 0, 1628, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 4094, 4094, 0, 4043, 430, 0, 1, 0, 1, 0, 551, 0, 3701, 0, 633, 0, 0, 0, 0, 2, 1, 2392] payload_rows_per_dst=[4872, 8190, 8190, 3258, 8190, 8176, 7624, 8124] dup_rows_per_dst=[8, 0, 1634, 0, 4, 770, 884, 5612] combine_rows_per_src=[4872, 8190, 8190, 3258, 8190, 8176, 7624, 8124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=189325312 dispatch_meta_bytes=228792 combine_payload_bytes=189325312 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=189554104 ingress_bytes=289988608 link_bytes=289988608 token_bytes=378879416 weight_movement_bytes=100663296 total_nvlink_bytes=479542712 weight_to_token_ratio=0.265687 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.206478244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55486 payload_rows_remote=47460 dup_rows=10050 dup_rows_remote=9578 dedup_ratio=0.153351 router_tokens_per_expert=[1, 14, 0, 0, 2218, 2, 0, 2468, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 2, 0, 0, 486, 0, 0, 0, 0, 0, 0, 0, 2, 0, 282, 4013, 4096, 0, 4090, 736, 0, 0, 0, 0, 14, 2468, 0, 3881, 0, 1709, 0, 2, 0, 0, 0, 0, 1956] payload_rows_per_dst=[5932, 8184, 8026, 6816, 8192, 3460, 8086, 6790] dup_rows_per_dst=[1476, 0, 472, 468, 32, 420, 4640, 2542] combine_rows_per_src=[5932, 8184, 8026, 6816, 8192, 3460, 8086, 6790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51124 payload_rows_remote=43850 dup_rows=14412 dup_rows_remote=14386 dedup_ratio=0.219910 router_tokens_per_expert=[0, 1, 0, 0, 3805, 0, 0, 3635, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 0, 0, 0, 4, 0, 0, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 2771, 0, 0, 0, 0, 0, 677, 0, 2825, 0, 3, 0, 0, 0, 0, 0, 0, 12] payload_rows_per_dst=[7612, 8122, 1510, 3802, 8192, 8192, 6420, 7274] dup_rows_per_dst=[0, 0, 0, 2, 0, 13730, 654, 26] combine_rows_per_src=[7612, 8122, 1510, 3802, 8192, 8192, 6420, 7274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45234 payload_rows_remote=37076 dup_rows=20302 dup_rows_remote=7140 dedup_ratio=0.309784 router_tokens_per_expert=[0, 0, 0, 0, 3753, 0, 0, 3380, 4084, 12, 0, 0, 37, 0, 4, 0, 64, 0, 0, 2, 0, 0, 0, 107, 2, 0, 1, 0, 14, 0, 0, 3310, 2, 0, 0, 4, 7, 0, 0, 0, 0, 0, 4079, 4083, 0, 4072, 359, 0, 0, 12, 12, 1, 343, 0, 3863, 1, 984, 2, 4, 0, 2, 0, 1, 167] payload_rows_per_dst=[8158, 8192, 8188, 8182, 8180, 718, 1346, 2270] dup_rows_per_dst=[13162, 82, 316, 6616, 12, 0, 64, 50] combine_rows_per_src=[8158, 8192, 8188, 8182, 8180, 718, 1346, 2270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=211345408 dispatch_meta_bytes=217232 combine_payload_bytes=211345408 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=211562640 ingress_bytes=312008704 link_bytes=312008704 token_bytes=422908048 weight_movement_bytes=100663296 total_nvlink_bytes=523571344 weight_to_token_ratio=0.238026 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17152 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.374025731e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=191660032 dispatch_meta_bytes=229368 combine_payload_bytes=191660032 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=191889400 ingress_bytes=216825856 link_bytes=216825856 token_bytes=383549432 weight_movement_bytes=25165824 total_nvlink_bytes=408715256 weight_to_token_ratio=0.065613 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.649794236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=194396160 dispatch_meta_bytes=228152 combine_payload_bytes=194396160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=194624312 ingress_bytes=219561984 link_bytes=219561984 token_bytes=389020472 weight_movement_bytes=25165824 total_nvlink_bytes=414186296 weight_to_token_ratio=0.064690 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27136 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.670613008e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=198385664 dispatch_meta_bytes=229368 combine_payload_bytes=198385664 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=198615032 ingress_bytes=248717312 link_bytes=248717312 token_bytes=397000696 weight_movement_bytes=50331648 total_nvlink_bytes=447332344 weight_to_token_ratio=0.126780 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.892451367e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=179609600 dispatch_meta_bytes=232944 combine_payload_bytes=179609600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=179842544 ingress_bytes=204775424 link_bytes=204775424 token_bytes=359452144 weight_movement_bytes=25165824 total_nvlink_bytes=384617968 weight_to_token_ratio=0.070012 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.558104372e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=151863296 dispatch_meta_bytes=176864 combine_payload_bytes=151863296 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=152040160 ingress_bytes=202194944 link_bytes=202194944 token_bytes=303903456 weight_movement_bytes=50331648 total_nvlink_bytes=354235104 weight_to_token_ratio=0.165617 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20480 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.538469900e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63230 payload_rows_remote=55880 dup_rows=2306 dup_rows_remote=1174 dedup_ratio=0.035187 router_tokens_per_expert=[0, 0, 0, 0, 0, 82, 2, 0, 0, 0, 0, 4095, 0, 48, 0, 0, 1, 10, 3716, 0, 0, 8, 0, 0, 0, 301, 1, 0, 0, 1104, 0, 0, 2835, 0, 0, 0, 0, 7, 3878, 1, 1, 6, 1, 0, 4073, 75, 1, 4025, 0, 0, 4071, 4, 0, 0, 0, 0, 3972, 0, 0, 449, 0, 0, 0, 1] payload_rows_per_dst=[8160, 8192, 7464, 7350, 7772, 8148, 8150, 7994] dup_rows_per_dst=[154, 94, 6, 1132, 0, 70, 0, 850] combine_rows_per_src=[8160, 8192, 7464, 7350, 7772, 8148, 8150, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57452 payload_rows_remote=50958 dup_rows=8084 dup_rows_remote=7926 dedup_ratio=0.123352 router_tokens_per_expert=[0, 0, 3, 2, 0, 153, 0, 0, 1, 2, 1, 4094, 1, 198, 0, 0, 2, 2112, 3331, 66, 0, 0, 2, 0, 0, 1, 3, 0, 0, 1505, 0, 0, 3423, 0, 0, 0, 0, 0, 1807, 0, 0, 0, 0, 0, 3655, 371, 2, 2953, 0, 0, 3856, 0, 0, 0, 0, 0, 4085, 3, 0, 1134, 0, 0, 2, 0] payload_rows_per_dst=[7474, 8192, 8006, 7788, 3614, 6494, 7712, 8172] dup_rows_per_dst=[152, 402, 3020, 2076, 0, 158, 0, 2276] combine_rows_per_src=[7474, 8192, 8006, 7788, 3614, 6494, 7712, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=228884480 dispatch_meta_bytes=228216 combine_payload_bytes=228884480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229112696 ingress_bytes=254050304 link_bytes=254050304 token_bytes=457997176 weight_movement_bytes=25165824 total_nvlink_bytes=483163000 weight_to_token_ratio=0.054948 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.933029275e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59592 payload_rows_remote=51412 dup_rows=5944 dup_rows_remote=3718 dedup_ratio=0.090698 router_tokens_per_expert=[3, 0, 0, 0, 3763, 0, 0, 2278, 4081, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 3974, 4087, 0, 3983, 318, 0, 0, 0, 1, 0, 1328, 51, 3823, 0, 49, 0, 0, 0, 0, 0, 0, 356] payload_rows_per_dst=[7532, 8168, 6096, 8108, 8178, 8104, 8180, 5226] dup_rows_per_dst=[0, 2, 546, 0, 0, 3030, 2226, 140] combine_rows_per_src=[7532, 8168, 6096, 8108, 8178, 8104, 8180, 5226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=208723968 dispatch_meta_bytes=235536 combine_payload_bytes=208723968 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=208959504 ingress_bytes=284221440 link_bytes=284221440 token_bytes=417683472 weight_movement_bytes=75497472 total_nvlink_bytes=493180944 weight_to_token_ratio=0.180753 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=26496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.162596759e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59498 payload_rows_remote=51306 dup_rows=6038 dup_rows_remote=5634 dedup_ratio=0.092133 router_tokens_per_expert=[0, 0, 0, 2, 0, 872, 0, 0, 0, 0, 0, 4096, 0, 202, 0, 0, 0, 178, 3806, 0, 0, 0, 0, 0, 2, 20, 1, 0, 0, 2072, 0, 0, 3492, 0, 0, 0, 0, 6, 1929, 0, 0, 0, 0, 0, 4060, 805, 0, 3305, 0, 0, 4096, 0, 0, 0, 0, 0, 3821, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[8166, 8192, 7736, 7898, 3870, 7802, 8192, 7642] dup_rows_per_dst=[1702, 404, 232, 3276, 0, 418, 0, 6] combine_rows_per_src=[8166, 8192, 7736, 7898, 3870, 7802, 8192, 7642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59832 payload_rows_remote=51644 dup_rows=5704 dup_rows_remote=1334 dedup_ratio=0.087036 router_tokens_per_expert=[0, 0, 0, 247, 0, 45, 0, 0, 0, 0, 0, 4090, 0, 15, 0, 0, 0, 2236, 4040, 2, 0, 0, 1, 0, 7, 28, 0, 1, 0, 44, 0, 0, 3902, 0, 0, 0, 0, 5, 3195, 0, 0, 2, 0, 0, 4054, 266, 0, 4095, 0, 0, 4007, 0, 0, 0, 0, 0, 2157, 0, 1, 328, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8180, 8188, 7808, 6392, 8190, 8014, 4910] dup_rows_per_dst=[542, 30, 4370, 156, 8, 536, 0, 62] combine_rows_per_src=[8150, 8180, 8188, 7808, 6392, 8190, 8014, 4910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47476 payload_rows_remote=39312 dup_rows=18060 dup_rows_remote=17176 dedup_ratio=0.275574 router_tokens_per_expert=[0, 0, 0, 127, 0, 919, 0, 0, 0, 0, 0, 4090, 0, 19, 0, 0, 0, 174, 3645, 0, 0, 0, 0, 0, 3, 15, 0, 0, 0, 56, 0, 0, 3627, 0, 0, 0, 0, 0, 3305, 0, 0, 0, 0, 0, 3945, 281, 0, 4044, 0, 0, 4094, 0, 0, 0, 0, 0, 4068, 0, 0, 356, 0, 0, 0, 0] payload_rows_per_dst=[2216, 7208, 348, 5342, 7852, 8158, 8188, 8164] dup_rows_per_dst=[974, 32, 0, 118, 7990, 7284, 778, 884] combine_rows_per_src=[2216, 7208, 348, 5342, 7852, 8158, 8188, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58078 payload_rows_remote=49932 dup_rows=7458 dup_rows_remote=7454 dedup_ratio=0.113800 router_tokens_per_expert=[0, 3, 0, 2, 1, 0, 0, 0, 7, 2, 5, 4086, 0, 4, 0, 0, 2, 36, 3795, 0, 2, 0, 29, 0, 94, 38, 13, 0, 2, 196, 5, 0, 4038, 0, 1, 0, 0, 16, 4092, 11, 0, 0, 0, 0, 4069, 0, 4, 4063, 0, 2, 4057, 3, 0, 0, 8, 4, 4039, 0, 0, 34, 1, 4, 0, 0] payload_rows_per_dst=[8146, 8190, 8172, 8174, 8186, 3222, 8124, 5864] dup_rows_per_dst=[4, 18, 6722, 598, 54, 0, 24, 38] combine_rows_per_src=[8146, 8190, 8172, 8174, 8186, 3222, 8124, 5864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62436 payload_rows_remote=54726 dup_rows=3100 dup_rows_remote=3100 dedup_ratio=0.047302 router_tokens_per_expert=[0, 0, 2, 1210, 0, 26, 0, 0, 1, 0, 1, 4082, 0, 6, 0, 0, 0, 2, 4019, 5, 0, 0, 1, 0, 55, 36, 0, 0, 0, 747, 0, 0, 2510, 0, 0, 0, 0, 0, 3854, 1, 1, 0, 0, 0, 3889, 2, 0, 3818, 0, 0, 4092, 0, 0, 0, 0, 0, 4093, 0, 0, 314, 0, 0, 1, 0] payload_rows_per_dst=[8148, 8166, 8042, 6362, 7710, 7636, 8184, 8188] dup_rows_per_dst=[2106, 14, 12, 334, 0, 6, 0, 628] combine_rows_per_src=[8148, 8166, 8042, 6362, 7710, 7636, 8184, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=210583552 dispatch_meta_bytes=220520 combine_payload_bytes=210583552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=210804072 ingress_bytes=260915200 link_bytes=260915200 token_bytes=421387624 weight_movement_bytes=50331648 total_nvlink_bytes=471719272 weight_to_token_ratio=0.119443 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.985263201e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=210149376 dispatch_meta_bytes=227760 combine_payload_bytes=210149376 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=210377136 ingress_bytes=260481024 link_bytes=260481024 token_bytes=420526512 weight_movement_bytes=50331648 total_nvlink_bytes=470858160 weight_to_token_ratio=0.119687 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.981959623e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=211533824 dispatch_meta_bytes=211912 combine_payload_bytes=211533824 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=211745736 ingress_bytes=261865472 link_bytes=261865472 token_bytes=423279560 weight_movement_bytes=50331648 total_nvlink_bytes=473611208 weight_to_token_ratio=0.118909 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24064 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.992493673e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64204 payload_rows_remote=56012 dup_rows=1332 dup_rows_remote=1226 dedup_ratio=0.020325 router_tokens_per_expert=[1203, 0, 0, 0, 4034, 2530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3, 0, 214, 0, 2, 4083, 4084, 0, 52, 0, 4095, 0, 2, 0, 0, 4079, 12, 4088, 0, 0, 0, 161, 0, 0, 0, 0, 30, 0, 0, 10, 0, 0, 126, 3956, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[7472, 8068, 8192, 8192, 8192, 8186, 7734, 8168] dup_rows_per_dst=[192, 0, 406, 106, 12, 362, 252, 2] combine_rows_per_src=[7472, 8068, 8192, 8192, 8192, 8186, 7734, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=224157696 dispatch_meta_bytes=231304 combine_payload_bytes=224157696 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=224389000 ingress_bytes=274489344 link_bytes=274489344 token_bytes=448546696 weight_movement_bytes=50331648 total_nvlink_bytes=498878344 weight_to_token_ratio=0.112210 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.088546753e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=161021952 dispatch_meta_bytes=225952 combine_payload_bytes=161021952 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=161247904 ingress_bytes=211353600 link_bytes=211353600 token_bytes=322269856 weight_movement_bytes=50331648 total_nvlink_bytes=372601504 weight_to_token_ratio=0.156179 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.608156690e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=204521472 dispatch_meta_bytes=229544 combine_payload_bytes=204521472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=204751016 ingress_bytes=229687296 link_bytes=229687296 token_bytes=409272488 weight_movement_bytes=25165824 total_nvlink_bytes=434438312 weight_to_token_ratio=0.061489 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=28544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.747654934e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58954 payload_rows_remote=50766 dup_rows=6582 dup_rows_remote=6534 dedup_ratio=0.100433 router_tokens_per_expert=[8, 0, 0, 0, 4052, 1229, 0, 1, 0, 0, 5, 0, 159, 307, 11, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 4087, 4094, 0, 1, 0, 3716, 11, 0, 0, 0, 4095, 0, 4089, 0, 0, 4, 3, 4, 0, 0, 20, 0, 14, 0, 3, 0, 0, 2733, 4021, 1, 0, 23, 0, 0, 0, 55, 0, 0] payload_rows_per_dst=[2476, 8192, 8190, 7436, 8180, 8188, 8100, 8192] dup_rows_per_dst=[0, 876, 40, 2, 20, 48, 5444, 152] combine_rows_per_src=[2476, 8192, 8190, 7436, 8180, 8188, 8100, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=229425152 dispatch_meta_bytes=228952 combine_payload_bytes=229425152 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=229654104 ingress_bytes=330088448 link_bytes=330088448 token_bytes=459079256 weight_movement_bytes=100663296 total_nvlink_bytes=559742552 weight_to_token_ratio=0.219272 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25856 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.511591693e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=207937536 dispatch_meta_bytes=229200 combine_payload_bytes=207937536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=208166736 ingress_bytes=233103360 link_bytes=233103360 token_bytes=416104272 weight_movement_bytes=25165824 total_nvlink_bytes=441270096 weight_to_token_ratio=0.060480 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.773647234e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55966 payload_rows_remote=47796 dup_rows=9570 dup_rows_remote=9570 dedup_ratio=0.146027 router_tokens_per_expert=[0, 0, 0, 87, 0, 538, 0, 0, 0, 0, 0, 4092, 0, 18, 1, 0, 0, 181, 4062, 78, 0, 0, 0, 0, 5, 39, 0, 0, 0, 945, 0, 0, 2573, 0, 0, 0, 0, 0, 3332, 0, 0, 10, 0, 0, 4096, 5, 0, 3692, 0, 0, 4085, 0, 0, 0, 0, 0, 3697, 0, 0, 1232, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 2738, 6660, 6718, 7408, 8170, 7894] dup_rows_per_dst=[1250, 36, 480, 464, 5370, 6, 0, 1964] combine_rows_per_src=[8192, 8186, 2738, 6660, 6718, 7408, 8170, 7894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59394 payload_rows_remote=51228 dup_rows=6142 dup_rows_remote=5648 dedup_ratio=0.093719 router_tokens_per_expert=[1207, 0, 58, 0, 4047, 36, 0, 0, 0, 0, 0, 0, 276, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 4096, 0, 1958, 0, 3469, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 129, 4, 0, 0, 0, 0, 0, 1072, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8166, 8192, 8128, 8192, 8192, 2144, 8192] dup_rows_per_dst=[2590, 494, 32, 2726, 0, 300, 0, 0] combine_rows_per_src=[8188, 8166, 8192, 8128, 8192, 8192, 2144, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58654 payload_rows_remote=50468 dup_rows=6882 dup_rows_remote=6880 dedup_ratio=0.105011 router_tokens_per_expert=[142, 0, 3, 0, 3822, 2670, 0, 0, 0, 0, 0, 0, 84, 664, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4081, 4094, 0, 98, 1, 3939, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 13, 0, 61, 1, 15, 15, 4, 0, 0, 2, 1, 823, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 8186, 7880, 8186, 8192, 1654, 8188] dup_rows_per_dst=[5534, 964, 2, 196, 0, 180, 6, 0] combine_rows_per_src=[8192, 8176, 8186, 7880, 8186, 8192, 1654, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59604 payload_rows_remote=51438 dup_rows=5932 dup_rows_remote=5932 dedup_ratio=0.090515 router_tokens_per_expert=[10, 0, 4, 0, 4094, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4085, 4096, 0, 285, 1, 3699, 0, 0, 0, 0, 4096, 0, 4082, 1, 0, 0, 1117, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2864, 8188, 8192, 7630, 8166, 8190, 8182, 8192] dup_rows_per_dst=[4, 0, 4, 340, 0, 2216, 3368, 0] combine_rows_per_src=[2864, 8188, 8192, 7630, 8166, 8190, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53448 payload_rows_remote=45256 dup_rows=12088 dup_rows_remote=12088 dedup_ratio=0.184448 router_tokens_per_expert=[329, 0, 0, 0, 3219, 1833, 0, 0, 0, 0, 0, 0, 1765, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 3916, 4096, 0, 2, 0, 4071, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 105, 0, 7, 0, 2, 0, 0, 0, 0, 0, 0, 42, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4320, 8128, 7364, 1284, 8184, 7950, 8026, 8192] dup_rows_per_dst=[4, 3836, 372, 2, 828, 110, 6936, 0] combine_rows_per_src=[4320, 8128, 7364, 1284, 8184, 7950, 8026, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48846 payload_rows_remote=40656 dup_rows=16690 dup_rows_remote=7608 dedup_ratio=0.254669 router_tokens_per_expert=[28, 0, 1, 6, 3772, 1805, 0, 5, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 4088, 0, 925, 0, 3264, 50, 0, 8, 0, 4081, 0, 4063, 0, 0, 0, 114, 7, 0, 8, 0, 6, 0, 0, 0, 2, 0, 2320, 4087, 2, 4, 0, 4, 0, 0, 4, 7, 0] payload_rows_per_dst=[8190, 2166, 8164, 8182, 7450, 1994, 4648, 8052] dup_rows_per_dst=[9082, 0, 0, 7546, 2, 46, 0, 14] combine_rows_per_src=[8190, 2166, 8164, 8182, 7450, 1994, 4648, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=195772416 dispatch_meta_bytes=229464 combine_payload_bytes=195772416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=196001880 ingress_bytes=220938240 link_bytes=220938240 token_bytes=391774296 weight_movement_bytes=25165824 total_nvlink_bytes=416940120 weight_to_token_ratio=0.064236 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.681084726e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=209829888 dispatch_meta_bytes=227504 combine_payload_bytes=209829888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210057392 ingress_bytes=234995712 link_bytes=234995712 token_bytes=419887280 weight_movement_bytes=25165824 total_nvlink_bytes=445053104 weight_to_token_ratio=0.059935 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788045846e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58964 payload_rows_remote=50776 dup_rows=6572 dup_rows_remote=6572 dedup_ratio=0.100281 router_tokens_per_expert=[0, 2, 1, 3911, 0, 0, 1356, 1, 0, 2, 12, 0, 0, 0, 669, 0, 0, 0, 2, 0, 0, 0, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 4069, 2854, 0, 0, 0, 0, 231, 0, 6, 0, 2387, 1026, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 3395, 568] payload_rows_per_dst=[7950, 8154, 8190, 8188, 6202, 5942, 8140, 6198] dup_rows_per_dst=[3728, 1362, 20, 0, 104, 1358, 0, 0] combine_rows_per_src=[7950, 8154, 8190, 8188, 6202, 5942, 8140, 6198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=206716928 dispatch_meta_bytes=229392 combine_payload_bytes=206716928 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206946320 ingress_bytes=231882752 link_bytes=231882752 token_bytes=413663248 weight_movement_bytes=25165824 total_nvlink_bytes=438829072 weight_to_token_ratio=0.060836 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.764359817e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=210690048 dispatch_meta_bytes=229480 combine_payload_bytes=210690048 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=210919528 ingress_bytes=261021696 link_bytes=261021696 token_bytes=421609576 weight_movement_bytes=50331648 total_nvlink_bytes=471941224 weight_to_token_ratio=0.119380 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.986073513e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=185368576 dispatch_meta_bytes=229376 combine_payload_bytes=185368576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=185597952 ingress_bytes=210534400 link_bytes=210534400 token_bytes=370966528 weight_movement_bytes=25165824 total_nvlink_bytes=396132352 weight_to_token_ratio=0.067839 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601923525e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=166526976 dispatch_meta_bytes=193056 combine_payload_bytes=166526976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=166720032 ingress_bytes=216858624 link_bytes=216858624 token_bytes=333247008 weight_movement_bytes=50331648 total_nvlink_bytes=383578656 weight_to_token_ratio=0.151034 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=12416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.650043562e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57374 payload_rows_remote=49300 dup_rows=8162 dup_rows_remote=7460 dedup_ratio=0.124542 router_tokens_per_expert=[1, 0, 0, 3903, 0, 0, 157, 4, 0, 0, 672, 0, 0, 1, 301, 0, 0, 0, 0, 0, 4, 6, 0, 4094, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 3673, 4094, 533, 0, 0, 0, 0, 108, 0, 0, 0, 4025, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 3900, 3038] payload_rows_per_dst=[7848, 7820, 8192, 7992, 1068, 8074, 8188, 8192] dup_rows_per_dst=[282, 1474, 16, 4, 0, 702, 0, 5684] combine_rows_per_src=[7848, 7820, 8192, 7992, 1068, 8074, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=207978496 dispatch_meta_bytes=229392 combine_payload_bytes=207978496 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=208207888 ingress_bytes=258310144 link_bytes=258310144 token_bytes=416186384 weight_movement_bytes=50331648 total_nvlink_bytes=466518032 weight_to_token_ratio=0.120935 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.965441735e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61052 payload_rows_remote=52874 dup_rows=4484 dup_rows_remote=3620 dedup_ratio=0.068420 router_tokens_per_expert=[2372, 0, 1, 0, 4063, 841, 0, 0, 0, 0, 0, 0, 47, 31, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 35, 0, 2908, 26, 0, 0, 0, 4096, 0, 4036, 0, 0, 0, 1478, 0, 0, 0, 118, 0, 0, 0, 1, 0, 0, 437, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6136, 8186, 8192, 5852, 8124, 8192, 8178, 8192] dup_rows_per_dst=[292, 96, 8, 34, 0, 3190, 864, 0] combine_rows_per_src=[6136, 8186, 8192, 5852, 8124, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=201932800 dispatch_meta_bytes=227040 combine_payload_bytes=201932800 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=202159840 ingress_bytes=252264448 link_bytes=252264448 token_bytes=404092640 weight_movement_bytes=50331648 total_nvlink_bytes=454424288 weight_to_token_ratio=0.124555 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.919440974e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56808 payload_rows_remote=48622 dup_rows=8728 dup_rows_remote=8460 dedup_ratio=0.133179 router_tokens_per_expert=[0, 0, 0, 2501, 0, 0, 184, 0, 0, 0, 187, 0, 0, 0, 27, 0, 0, 0, 0, 0, 32, 0, 9, 4078, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 4013, 4096, 2113, 1, 0, 0, 0, 0, 0, 0, 0, 3888, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 3920, 3574] payload_rows_per_dst=[8134, 8186, 8182, 8172, 8166, 7776, 8192, 0] dup_rows_per_dst=[4384, 268, 56, 10, 3902, 108, 0, 0] combine_rows_per_src=[8134, 8186, 8182, 8172, 8166, 7776, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55826 payload_rows_remote=55568 dup_rows=9710 dup_rows_remote=9708 dedup_ratio=0.148163 router_tokens_per_expert=[0, 0, 0, 2978, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 28, 1, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4, 0, 4089, 4091, 122, 4, 0, 0, 0, 1227, 0, 4, 0, 4031, 83, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4025, 3853] payload_rows_per_dst=[6558, 8184, 8192, 8192, 258, 8192, 8190, 8060] dup_rows_per_dst=[162, 68, 4, 24, 2, 2498, 6, 6946] combine_rows_per_src=[6558, 8184, 8192, 8192, 258, 8192, 8190, 8060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57030 payload_rows_remote=48838 dup_rows=8506 dup_rows_remote=8502 dedup_ratio=0.129791 router_tokens_per_expert=[0, 0, 0, 1233, 7, 0, 192, 0, 0, 0, 23, 0, 0, 0, 317, 0, 0, 0, 5, 0, 0, 0, 0, 4093, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2, 0, 3824, 4087, 4034, 0, 0, 0, 0, 49, 0, 0, 0, 4078, 0, 1, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3703, 3010] payload_rows_per_dst=[8042, 8082, 8192, 8186, 8182, 8156, 8190, 0] dup_rows_per_dst=[842, 246, 4, 10, 7296, 98, 10, 0] combine_rows_per_src=[8042, 8082, 8192, 8186, 8182, 8156, 8190, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53384 payload_rows_remote=45192 dup_rows=12152 dup_rows_remote=3356 dedup_ratio=0.185425 router_tokens_per_expert=[0, 0, 0, 420, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3901, 4096, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 3912] payload_rows_per_dst=[844, 4442, 8118, 8192, 8146, 7258, 8192, 8192] dup_rows_per_dst=[0, 2, 24, 2, 3222, 2, 104, 8796] combine_rows_per_src=[844, 4442, 8118, 8192, 8146, 7258, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56510 payload_rows_remote=48722 dup_rows=9026 dup_rows_remote=8118 dedup_ratio=0.137726 router_tokens_per_expert=[1, 1, 6, 3787, 1, 0, 42, 0, 0, 4, 172, 0, 2, 2, 145, 18, 0, 11, 0, 0, 0, 0, 0, 4088, 0, 8, 0, 0, 0, 3, 6, 12, 0, 0, 0, 4028, 4068, 3988, 0, 0, 5, 0, 941, 21, 45, 0, 3165, 364, 0, 0, 0, 7, 0, 3, 1, 1, 0, 2, 4078, 0, 0, 0, 3232, 510] payload_rows_per_dst=[7788, 8184, 8186, 8184, 8176, 7836, 8152, 4] dup_rows_per_dst=[908, 558, 12, 134, 6160, 1246, 8, 0] combine_rows_per_src=[7788, 8184, 8186, 8184, 8176, 7836, 8152, 4] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=216571904 dispatch_meta_bytes=225976 combine_payload_bytes=216571904 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=216797880 ingress_bytes=266903552 link_bytes=266903552 token_bytes=433369784 weight_movement_bytes=50331648 total_nvlink_bytes=483701432 weight_to_token_ratio=0.116140 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.030827641e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56780 payload_rows_remote=51760 dup_rows=8756 dup_rows_remote=8688 dedup_ratio=0.133606 router_tokens_per_expert=[0, 3, 1, 1853, 1, 0, 4095, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 1, 0, 4087, 0, 0, 0, 4094, 0, 0, 3920, 1, 2237, 0, 0, 1, 0, 0, 0, 3069, 3750, 0, 0, 10, 0, 4, 0, 131, 19, 0, 0, 0, 0, 1468, 0, 0, 7, 1, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8022, 8190, 5020, 8176, 8062, 2938, 8180] dup_rows_per_dst=[3714, 0, 0, 68, 4872, 86, 14, 2] combine_rows_per_src=[8192, 8022, 8190, 5020, 8176, 8062, 2938, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=199155712 dispatch_meta_bytes=228328 combine_payload_bytes=199155712 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199384040 ingress_bytes=224321536 link_bytes=224321536 token_bytes=398539752 weight_movement_bytes=25165824 total_nvlink_bytes=423705576 weight_to_token_ratio=0.063145 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.706827700e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=227606528 dispatch_meta_bytes=261104 combine_payload_bytes=227606528 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=227867632 ingress_bytes=328269824 link_bytes=328269824 token_bytes=455474160 weight_movement_bytes=100663296 total_nvlink_bytes=556137456 weight_to_token_ratio=0.221008 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=20096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.497754066e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=200040448 dispatch_meta_bytes=229360 combine_payload_bytes=200040448 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=200269808 ingress_bytes=225206272 link_bytes=225206272 token_bytes=400310256 weight_movement_bytes=25165824 total_nvlink_bytes=425476080 weight_to_token_ratio=0.062866 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.713559518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=185106432 dispatch_meta_bytes=194192 combine_payload_bytes=185106432 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=185300624 ingress_bytes=310935552 link_bytes=310935552 token_bytes=370407056 weight_movement_bytes=125829120 total_nvlink_bytes=496236176 weight_to_token_ratio=0.339705 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=19200 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.365860285e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=199565312 dispatch_meta_bytes=227360 combine_payload_bytes=199565312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199792672 ingress_bytes=224731136 link_bytes=224731136 token_bytes=399357984 weight_movement_bytes=25165824 total_nvlink_bytes=424523808 weight_to_token_ratio=0.063016 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.709944282e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57640 payload_rows_remote=49448 dup_rows=7896 dup_rows_remote=7884 dedup_ratio=0.120483 router_tokens_per_expert=[0, 0, 0, 904, 0, 0, 4079, 0, 51, 231, 0, 0, 4096, 0, 2, 0, 21, 0, 0, 4094, 0, 0, 0, 4091, 2, 0, 4093, 5, 3964, 0, 0, 0, 0, 0, 2, 1745, 2023, 2, 0, 0, 0, 0, 0, 6, 0, 3, 0, 0, 0, 3354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1808, 8192, 8182, 7938, 6948, 8192, 8192, 8188] dup_rows_per_dst=[0, 568, 42, 4, 596, 12, 6674, 0] combine_rows_per_src=[1808, 8192, 8182, 7938, 6948, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=212008960 dispatch_meta_bytes=241792 combine_payload_bytes=212008960 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=212250752 ingress_bytes=287506432 link_bytes=287506432 token_bytes=424259712 weight_movement_bytes=75497472 total_nvlink_bytes=499757184 weight_to_token_ratio=0.177951 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.187591752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59670 payload_rows_remote=51506 dup_rows=5866 dup_rows_remote=5866 dedup_ratio=0.089508 router_tokens_per_expert=[0, 0, 0, 3965, 0, 0, 34, 34, 0, 0, 10, 0, 0, 0, 264, 0, 0, 0, 0, 0, 17, 0, 0, 4057, 1, 0, 0, 6, 38, 0, 0, 0, 0, 0, 0, 3979, 4082, 2326, 0, 0, 0, 0, 74, 0, 0, 0, 3025, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4014, 2718] payload_rows_per_dst=[7952, 8144, 8148, 8184, 4652, 6238, 8164, 8188] dup_rows_per_dst=[114, 362, 0, 62, 0, 52, 0, 5276] combine_rows_per_src=[7952, 8144, 8148, 8184, 4652, 6238, 8164, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=202539008 dispatch_meta_bytes=229328 combine_payload_bytes=202539008 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=202768336 ingress_bytes=252870656 link_bytes=252870656 token_bytes=405307344 weight_movement_bytes=50331648 total_nvlink_bytes=455638992 weight_to_token_ratio=0.124181 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.924053516e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56822 payload_rows_remote=48630 dup_rows=8714 dup_rows_remote=8246 dedup_ratio=0.132965 router_tokens_per_expert=[0, 40, 0, 301, 0, 0, 4096, 0, 217, 17, 0, 0, 4096, 0, 0, 0, 64, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 0, 3719, 0, 0, 0, 0, 0, 2, 247, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 498, 8186, 7182, 8192] dup_rows_per_dst=[682, 468, 128, 7432, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 498, 8186, 7182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59658 payload_rows_remote=51468 dup_rows=5878 dup_rows_remote=898 dedup_ratio=0.089691 router_tokens_per_expert=[0, 9, 0, 2846, 0, 1, 4095, 5, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 4095, 1, 0, 4096, 0, 2531, 0, 0, 0, 0, 0, 0, 2498, 4087, 0, 0, 0, 1, 0, 0, 17, 1, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6488, 8192, 8192, 5064, 8190, 8192, 7150, 8190] dup_rows_per_dst=[288, 2, 2, 0, 4980, 38, 568, 0] combine_rows_per_src=[6488, 8192, 8192, 5064, 8190, 8192, 7150, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56392 payload_rows_remote=48204 dup_rows=9144 dup_rows_remote=9056 dedup_ratio=0.139526 router_tokens_per_expert=[0, 15, 0, 3651, 0, 2, 4096, 42, 75, 20, 0, 0, 4083, 0, 0, 0, 44, 0, 0, 4090, 0, 0, 0, 4094, 0, 1, 4096, 0, 667, 0, 0, 0, 0, 0, 0, 69, 4065, 0, 0, 0, 0, 0, 0, 43, 21, 0, 0, 0, 0, 3588, 0, 0, 0, 0, 2, 1, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8176, 8188, 8138, 138, 8192, 7182, 8186] dup_rows_per_dst=[7420, 180, 88, 1328, 0, 128, 0, 0] combine_rows_per_src=[8192, 8176, 8188, 8138, 138, 8192, 7182, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55994 payload_rows_remote=47802 dup_rows=9542 dup_rows_remote=9532 dedup_ratio=0.145599 router_tokens_per_expert=[0, 3, 0, 267, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4046, 0, 0, 0, 0, 0, 0, 1182, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3427, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[538, 6842, 8182, 7696, 8160, 8192, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 2230, 396, 6904, 10] combine_rows_per_src=[538, 6842, 8182, 7696, 8160, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=210968576 dispatch_meta_bytes=229488 combine_payload_bytes=210968576 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=211198064 ingress_bytes=261300224 link_bytes=261300224 token_bytes=422166640 weight_movement_bytes=50331648 total_nvlink_bytes=472498288 weight_to_token_ratio=0.119222 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.988192789e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55670 payload_rows_remote=47480 dup_rows=9866 dup_rows_remote=1876 dedup_ratio=0.150543 router_tokens_per_expert=[0, 0, 11, 1520, 0, 8, 4080, 0, 0, 0, 0, 0, 4053, 0, 14, 0, 84, 4, 0, 4084, 0, 0, 0, 4075, 0, 0, 3741, 0, 3382, 11, 0, 9, 2, 0, 1, 3064, 3641, 9, 2, 0, 10, 0, 0, 120, 4, 0, 5, 0, 0, 647, 0, 2, 41, 7, 0, 119, 0, 0, 3, 0, 6, 0, 9, 0] payload_rows_per_dst=[8190, 8178, 7942, 7696, 6152, 7744, 1576, 8192] dup_rows_per_dst=[7990, 326, 14, 1448, 4, 16, 56, 12] combine_rows_per_src=[8190, 8178, 7942, 7696, 6152, 7744, 1576, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60018 payload_rows_remote=51836 dup_rows=5518 dup_rows_remote=5428 dedup_ratio=0.084198 router_tokens_per_expert=[0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 5, 0, 2152, 1, 0, 3423, 50, 2, 2966, 1, 0, 0, 0, 4088, 3, 0, 45, 0, 0, 0, 0, 56, 0, 2, 1, 2, 3, 0, 0, 0, 306, 0, 4094, 0, 0, 0, 1, 3321, 0, 4055, 0, 0, 0, 0, 4089, 4087, 0, 2, 0, 0, 2, 0, 0] payload_rows_per_dst=[8178, 4312, 8160, 8182, 8176, 8190, 6642, 8178] dup_rows_per_dst=[22, 4, 4722, 90, 62, 612, 2, 4] combine_rows_per_src=[8178, 4312, 8160, 8182, 8176, 8190, 6642, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=199188480 dispatch_meta_bytes=227504 combine_payload_bytes=199188480 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=199415984 ingress_bytes=249520128 link_bytes=249520128 token_bytes=398604464 weight_movement_bytes=50331648 total_nvlink_bytes=448936112 weight_to_token_ratio=0.126270 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.898559870e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=210812928 dispatch_meta_bytes=209464 combine_payload_bytes=210812928 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=211022392 ingress_bytes=286310400 link_bytes=286310400 token_bytes=421835320 weight_movement_bytes=75497472 total_nvlink_bytes=497332792 weight_to_token_ratio=0.178974 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.178491331e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=197443584 dispatch_meta_bytes=229040 combine_payload_bytes=197443584 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=197672624 ingress_bytes=272941056 link_bytes=272941056 token_bytes=395116208 weight_movement_bytes=75497472 total_nvlink_bytes=470613680 weight_to_token_ratio=0.191077 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.076766070e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62660 payload_rows_remote=54476 dup_rows=2876 dup_rows_remote=2450 dedup_ratio=0.043884 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3344, 0, 0, 3413, 3, 0, 1201, 4, 1, 5, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 2, 0, 0, 213, 0, 4092, 0, 0, 0, 0, 4095, 0, 4088, 1, 0, 0, 1, 4090, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 6690, 6840, 8182, 8192, 8184, 8192, 8192] dup_rows_per_dst=[0, 12, 12, 2, 20, 426, 2404, 0] combine_rows_per_src=[8188, 6690, 6840, 8182, 8192, 8184, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=195796992 dispatch_meta_bytes=229336 combine_payload_bytes=195796992 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=196026328 ingress_bytes=246128640 link_bytes=246128640 token_bytes=391823320 weight_movement_bytes=50331648 total_nvlink_bytes=442154968 weight_to_token_ratio=0.128455 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.872754564e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=194478080 dispatch_meta_bytes=197424 combine_payload_bytes=194478080 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=194675504 ingress_bytes=244809728 link_bytes=244809728 token_bytes=389153584 weight_movement_bytes=50331648 total_nvlink_bytes=439485232 weight_to_token_ratio=0.129336 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.862719168e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=212320256 dispatch_meta_bytes=229056 combine_payload_bytes=212320256 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=212549312 ingress_bytes=262651904 link_bytes=262651904 token_bytes=424869568 weight_movement_bytes=50331648 total_nvlink_bytes=475201216 weight_to_token_ratio=0.118464 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.998477512e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58414 payload_rows_remote=50222 dup_rows=7122 dup_rows_remote=6410 dedup_ratio=0.108673 router_tokens_per_expert=[0, 0, 0, 1907, 0, 0, 4096, 13, 3, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4050, 0, 2758, 0, 0, 0, 0, 0, 0, 3387, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3840, 8192, 8192, 5516, 8190, 8100, 8192, 8192] dup_rows_per_dst=[0, 6, 2, 0, 6402, 0, 712, 0] combine_rows_per_src=[3840, 8192, 8192, 5516, 8190, 8100, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=223133696 dispatch_meta_bytes=227704 combine_payload_bytes=223133696 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=223361400 ingress_bytes=273465344 link_bytes=273465344 token_bytes=446495096 weight_movement_bytes=50331648 total_nvlink_bytes=496826744 weight_to_token_ratio=0.112726 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.080755296e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61682 payload_rows_remote=53490 dup_rows=3854 dup_rows_remote=86 dedup_ratio=0.058807 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 3174, 0, 0, 2806, 0, 0, 2646, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4096, 0, 0, 0, 0, 3837, 0, 3879, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5292, 8190, 7758, 8192, 7674, 8192] dup_rows_per_dst=[30, 3768, 0, 0, 0, 56, 0, 0] combine_rows_per_src=[8192, 8192, 5292, 8190, 7758, 8192, 7674, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=205709312 dispatch_meta_bytes=226528 combine_payload_bytes=205709312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=205935840 ingress_bytes=230875136 link_bytes=230875136 token_bytes=411645152 weight_movement_bytes=25165824 total_nvlink_bytes=436810976 weight_to_token_ratio=0.061135 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.756693023e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59250 payload_rows_remote=51072 dup_rows=6286 dup_rows_remote=6272 dedup_ratio=0.095917 router_tokens_per_expert=[0, 1, 0, 0, 0, 4, 2, 0, 1, 0, 0, 0, 0, 1196, 0, 0, 4010, 33, 0, 1988, 0, 0, 1, 0, 4095, 0, 0, 1407, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3649, 0, 4088, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 2394, 8020, 8190, 8178, 8192, 7894, 8190] dup_rows_per_dst=[10, 0, 68, 2814, 14, 0, 3380, 0] combine_rows_per_src=[8192, 2394, 8020, 8190, 8178, 8192, 7894, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57288 payload_rows_remote=53372 dup_rows=8248 dup_rows_remote=8240 dedup_ratio=0.125854 router_tokens_per_expert=[0, 0, 0, 0, 0, 23, 0, 3, 0, 0, 0, 2, 0, 3948, 0, 0, 4036, 0, 0, 1939, 0, 0, 0, 0, 4093, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1, 4094, 0, 0, 0, 0, 2115, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 3916, 8186, 8192, 8190, 4230, 8190] dup_rows_per_dst=[52, 7734, 8, 250, 0, 204, 0, 0] combine_rows_per_src=[8192, 8192, 3916, 8186, 8192, 8190, 4230, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63352 payload_rows_remote=55160 dup_rows=2184 dup_rows_remote=2144 dedup_ratio=0.033325 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3342, 0, 0, 4002, 0, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 189, 0, 3924, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8106, 6684, 8004, 8192, 7848, 8152, 8174, 8192] dup_rows_per_dst=[0, 0, 0, 1704, 0, 12, 428, 40] combine_rows_per_src=[8106, 6684, 8004, 8192, 7848, 8152, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58872 payload_rows_remote=52610 dup_rows=6664 dup_rows_remote=6656 dedup_ratio=0.101685 router_tokens_per_expert=[0, 4089, 0, 1, 0, 0, 0, 2, 23, 1332, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 12, 0, 3, 0, 4093, 4082, 4094, 0, 0, 2, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 1309, 0, 2, 0, 4089, 0, 1690, 1, 0, 0, 4091, 0, 0, 3829, 1, 1, 0, 0, 0, 5] payload_rows_per_dst=[8182, 8190, 8188, 6262, 8182, 4010, 8188, 7670] dup_rows_per_dst=[2, 2710, 10, 8, 0, 556, 3376, 2] combine_rows_per_src=[8182, 8190, 8188, 6262, 8182, 4010, 8188, 7670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57818 payload_rows_remote=49658 dup_rows=7718 dup_rows_remote=7718 dedup_ratio=0.117767 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1049, 0, 2, 3905, 90, 15, 3122, 0, 3, 0, 0, 4078, 2, 0, 67, 0, 0, 0, 14, 0, 38, 2, 0, 8, 5, 6, 0, 0, 4, 0, 4084, 2, 0, 1, 7, 4020, 0, 4047, 11, 12, 3, 0, 4080, 4048, 0, 0, 6, 9, 10, 9, 1] payload_rows_per_dst=[8160, 8122, 7712, 3258, 8192, 6914, 7344, 8116] dup_rows_per_dst=[0, 1806, 5018, 66, 758, 10, 10, 50] combine_rows_per_src=[8160, 8122, 7712, 3258, 8192, 6914, 7344, 8116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=219095040 dispatch_meta_bytes=214304 combine_payload_bytes=219095040 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=219309344 ingress_bytes=244260864 link_bytes=244260864 token_bytes=438404384 weight_movement_bytes=25165824 total_nvlink_bytes=463570208 weight_to_token_ratio=0.057403 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.858542947e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57078 payload_rows_remote=48894 dup_rows=8458 dup_rows_remote=2028 dedup_ratio=0.129059 router_tokens_per_expert=[1, 4058, 5, 0, 0, 6, 0, 1, 1, 393, 1, 6, 0, 0, 0, 1, 1, 8, 0, 250, 0, 1, 0, 0, 3494, 4096, 4091, 4, 0, 0, 1, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 0, 0, 0, 4093, 0, 499, 5, 0, 0, 3870, 0, 0, 4034, 0, 0, 0, 2, 0, 7] payload_rows_per_dst=[8132, 6882, 8192, 1444, 8190, 8184, 7970, 8084] dup_rows_per_dst=[10, 668, 520, 2, 48, 6430, 778, 2] combine_rows_per_src=[8132, 6882, 8192, 1444, 8190, 8184, 7970, 8084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=209190912 dispatch_meta_bytes=229376 combine_payload_bytes=209190912 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=209420288 ingress_bytes=259522560 link_bytes=259522560 token_bytes=418611200 weight_movement_bytes=50331648 total_nvlink_bytes=468942848 weight_to_token_ratio=0.120235 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.974666820e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=218611712 dispatch_meta_bytes=246448 combine_payload_bytes=218611712 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=218858160 ingress_bytes=294109184 link_bytes=294109184 token_bytes=437469872 weight_movement_bytes=75497472 total_nvlink_bytes=512967344 weight_to_token_ratio=0.172578 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.237831066e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=225935360 dispatch_meta_bytes=229216 combine_payload_bytes=225935360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=226164576 ingress_bytes=251101184 link_bytes=251101184 token_bytes=452099936 weight_movement_bytes=25165824 total_nvlink_bytes=477265760 weight_to_token_ratio=0.055664 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.910589879e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=215490560 dispatch_meta_bytes=237064 combine_payload_bytes=215490560 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=215727624 ingress_bytes=316153856 link_bytes=316153856 token_bytes=431218184 weight_movement_bytes=100663296 total_nvlink_bytes=531881480 weight_to_token_ratio=0.233439 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=14848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.405565549e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=203399168 dispatch_meta_bytes=229504 combine_payload_bytes=203399168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203628672 ingress_bytes=228564992 link_bytes=228564992 token_bytes=407027840 weight_movement_bytes=25165824 total_nvlink_bytes=432193664 weight_to_token_ratio=0.061828 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.739115497e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64450 payload_rows_remote=56268 dup_rows=1086 dup_rows_remote=106 dedup_ratio=0.016571 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 9, 0, 0, 0, 0, 0, 12, 3801, 0, 0, 3878, 1, 0, 1590, 3, 0, 0, 0, 4093, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 2991, 0, 4073, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7602, 7758, 8186, 8146, 8192, 8182, 8192] dup_rows_per_dst=[22, 24, 6, 44, 0, 10, 980, 0] combine_rows_per_src=[8192, 7602, 7758, 8186, 8146, 8192, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=200269824 dispatch_meta_bytes=203688 combine_payload_bytes=200269824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200473512 ingress_bytes=225435648 link_bytes=225435648 token_bytes=400743336 weight_movement_bytes=25165824 total_nvlink_bytes=425909160 weight_to_token_ratio=0.062798 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.715304805e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56466 payload_rows_remote=48274 dup_rows=9070 dup_rows_remote=8218 dedup_ratio=0.138397 router_tokens_per_expert=[0, 3983, 0, 0, 0, 288, 0, 0, 14, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 4074, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 2922, 0, 0, 0, 4096, 0, 564, 0, 0, 0, 4061, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7974, 8192, 8192, 8156, 8192, 5844, 1724, 8192] dup_rows_per_dst=[568, 852, 46, 7440, 72, 0, 92, 0] combine_rows_per_src=[7974, 8192, 8192, 8156, 8192, 5844, 1724, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230473728 dispatch_meta_bytes=225496 combine_payload_bytes=230473728 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=230699224 ingress_bytes=331137024 link_bytes=331137024 token_bytes=461172952 weight_movement_bytes=100663296 total_nvlink_bytes=561836248 weight_to_token_ratio=0.218277 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.519570145e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56618 payload_rows_remote=48426 dup_rows=8918 dup_rows_remote=8824 dedup_ratio=0.136078 router_tokens_per_expert=[0, 3852, 7, 1, 0, 125, 0, 0, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 4096, 4090, 0, 0, 0, 2, 0, 0, 0, 47, 0, 0, 0, 0, 1, 1, 0, 0, 0, 230, 0, 0, 0, 4095, 0, 2479, 1, 0, 0, 4087, 0, 0, 4071, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7738, 8190, 8192, 4, 8192, 7970, 8190, 8142] dup_rows_per_dst=[232, 3188, 0, 0, 94, 460, 4944, 0] combine_rows_per_src=[7738, 8190, 8192, 4, 8192, 7970, 8190, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60514 payload_rows_remote=52326 dup_rows=5022 dup_rows_remote=5010 dedup_ratio=0.076630 router_tokens_per_expert=[0, 4096, 3, 3, 0, 304, 29, 0, 0, 733, 1, 23, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 3877, 4092, 4076, 0, 0, 0, 0, 1514, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 1728, 0, 0, 0, 4096, 0, 33, 2, 0, 0, 4028, 0, 0, 4087, 0, 0, 0, 0, 0, 7] payload_rows_per_dst=[8192, 8188, 8188, 8056, 8192, 3456, 8060, 8182] dup_rows_per_dst=[678, 1478, 12, 2726, 56, 0, 66, 6] combine_rows_per_src=[8192, 8188, 8188, 8056, 8192, 3456, 8060, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64770 payload_rows_remote=57066 dup_rows=766 dup_rows_remote=766 dedup_ratio=0.011688 router_tokens_per_expert=[0, 0, 0, 0, 4090, 6, 4091, 3965, 0, 0, 4091, 3, 0, 2, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 3847, 0, 3, 2, 0, 0, 24, 4083, 1, 0, 0, 0, 0, 4, 0, 0, 0, 4094, 0, 0, 5, 43, 0, 2, 0, 0, 2, 0, 0, 172, 0, 2, 0, 0, 0, 0, 139, 0] payload_rows_per_dst=[7942, 8188, 8190, 7704, 8176, 8192, 8192, 8186] dup_rows_per_dst=[0, 4, 4, 0, 48, 92, 340, 278] combine_rows_per_src=[7942, 8188, 8190, 7704, 8176, 8192, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54898 payload_rows_remote=46708 dup_rows=10638 dup_rows_remote=10050 dedup_ratio=0.162323 router_tokens_per_expert=[0, 4082, 1, 0, 0, 496, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3323, 4096, 3935, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3025, 0, 0, 0, 4096, 0, 933, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6576, 1158, 7668, 7878, 7616, 7624, 8188, 8190] dup_rows_per_dst=[886, 0, 4, 526, 14, 5072, 3548, 588] combine_rows_per_src=[6576, 1158, 7668, 7878, 7616, 7624, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=197730304 dispatch_meta_bytes=225968 combine_payload_bytes=197730304 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197956272 ingress_bytes=222896128 link_bytes=222896128 token_bytes=395686576 weight_movement_bytes=25165824 total_nvlink_bytes=420852400 weight_to_token_ratio=0.063600 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.695981992e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51124 payload_rows_remote=43050 dup_rows=14412 dup_rows_remote=14292 dedup_ratio=0.219910 router_tokens_per_expert=[1, 3988, 44, 0, 0, 54, 0, 10, 0, 3106, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 4056, 4084, 0, 0, 2, 24, 0, 8, 0, 158, 0, 0, 0, 0, 0, 0, 3, 16, 9, 518, 0, 0, 0, 4070, 0, 439, 195, 0, 0, 4066, 1, 24, 3908, 11, 0, 144, 7, 33, 17] payload_rows_per_dst=[8074, 8188, 8112, 8174, 8168, 1068, 1224, 8116] dup_rows_per_dst=[120, 6212, 0, 7534, 304, 24, 46, 172] combine_rows_per_src=[8074, 8188, 8112, 8174, 8168, 1068, 1224, 8116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=198352896 dispatch_meta_bytes=229000 combine_payload_bytes=198352896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198581896 ingress_bytes=223518720 link_bytes=223518720 token_bytes=396934792 weight_movement_bytes=25165824 total_nvlink_bytes=422100616 weight_to_token_ratio=0.063400 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.700719198e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64822 payload_rows_remote=56780 dup_rows=714 dup_rows_remote=714 dedup_ratio=0.010895 router_tokens_per_expert=[38, 0, 1, 0, 4093, 0, 4086, 3824, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 75, 0, 8, 4061, 2, 0, 0, 13, 2, 4037, 0, 0, 0, 19, 0, 232, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 2, 0, 0, 75, 0] payload_rows_per_dst=[7726, 8192, 8188, 8100, 8192, 8042, 8190, 8192] dup_rows_per_dst=[0, 2, 104, 42, 430, 0, 0, 136] combine_rows_per_src=[7726, 8192, 8188, 8100, 8192, 8042, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233742336 dispatch_meta_bytes=231328 combine_payload_bytes=233742336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233973664 ingress_bytes=258908160 link_bytes=258908160 token_bytes=467716000 weight_movement_bytes=25165824 total_nvlink_bytes=492881824 weight_to_token_ratio=0.053806 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30208 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.969991946e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=191315968 dispatch_meta_bytes=227032 combine_payload_bytes=191315968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=191543000 ingress_bytes=216481792 link_bytes=216481792 token_bytes=382858968 weight_movement_bytes=25165824 total_nvlink_bytes=408024792 weight_to_token_ratio=0.065731 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.647176306e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=214327296 dispatch_meta_bytes=229344 combine_payload_bytes=214327296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=214556640 ingress_bytes=239493120 link_bytes=239493120 token_bytes=428883936 weight_movement_bytes=25165824 total_nvlink_bytes=454049760 weight_to_token_ratio=0.058677 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.822265924e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57888 payload_rows_remote=49696 dup_rows=7648 dup_rows_remote=6698 dedup_ratio=0.116699 router_tokens_per_expert=[0, 4084, 0, 0, 0, 2, 0, 0, 0, 707, 0, 13, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4055, 4096, 3662, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 11, 0, 0, 0, 0, 3364, 0, 0, 0, 4094, 0, 462, 13, 0, 0, 4096, 0, 0, 4094, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8168, 1440, 8192, 7324, 8192, 8192, 8192, 8188] dup_rows_per_dst=[4, 0, 20, 2, 24, 6646, 950, 2] combine_rows_per_src=[8168, 1440, 8192, 7324, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=176332800 dispatch_meta_bytes=229368 combine_payload_bytes=176332800 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=176562168 ingress_bytes=226664448 link_bytes=226664448 token_bytes=352894968 weight_movement_bytes=50331648 total_nvlink_bytes=403226616 weight_to_token_ratio=0.142625 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.724654553e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232570880 dispatch_meta_bytes=229976 combine_payload_bytes=232570880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=232800856 ingress_bytes=257736704 link_bytes=257736704 token_bytes=465371736 weight_movement_bytes=25165824 total_nvlink_bytes=490537560 weight_to_token_ratio=0.054077 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.961078519e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62846 payload_rows_remote=54654 dup_rows=2690 dup_rows_remote=2690 dedup_ratio=0.041046 router_tokens_per_expert=[1, 0, 0, 0, 4096, 1, 4096, 3311, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 841, 0, 0, 2, 0, 3553, 0, 0, 0, 1, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0] payload_rows_per_dst=[6626, 8192, 8192, 7112, 8192, 8148, 8192, 8192] dup_rows_per_dst=[0, 0, 1712, 0, 52, 0, 0, 926] combine_rows_per_src=[6626, 8192, 8192, 7112, 8192, 8148, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65162 payload_rows_remote=57002 dup_rows=374 dup_rows_remote=340 dedup_ratio=0.005707 router_tokens_per_expert=[0, 0, 0, 0, 4095, 0, 4093, 3980, 0, 0, 4096, 0, 0, 0, 3, 1, 1, 1, 0, 0, 4095, 71, 0, 0, 290, 0, 3792, 0, 1, 0, 0, 0, 30, 4067, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 49, 0] payload_rows_per_dst=[7960, 8192, 8192, 8118, 8160, 8158, 8192, 8190] dup_rows_per_dst=[0, 8, 144, 48, 34, 2, 44, 94] combine_rows_per_src=[7960, 8192, 8192, 8118, 8160, 8158, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=203554816 dispatch_meta_bytes=225576 combine_payload_bytes=203554816 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=203780392 ingress_bytes=279052288 link_bytes=279052288 token_bytes=407335208 weight_movement_bytes=75497472 total_nvlink_bytes=482832680 weight_to_token_ratio=0.185345 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=24704 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.123265485e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60820 payload_rows_remote=52828 dup_rows=4716 dup_rows_remote=4712 dedup_ratio=0.071960 router_tokens_per_expert=[0, 774, 1, 4093, 0, 0, 0, 0, 2, 3138, 0, 0, 0, 0, 0, 2, 0, 4086, 1, 4081, 237, 0, 1, 2, 0, 3988, 0, 2, 0, 0, 0, 8, 0, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 41, 0, 4084, 2, 0, 4086, 0, 0, 0, 0, 0, 2, 3, 0, 0, 484, 0, 0, 0, 3637, 0] payload_rows_per_dst=[5398, 6744, 8188, 7992, 8188, 8192, 8180, 7938] dup_rows_per_dst=[1246, 2632, 456, 4, 6, 66, 2, 304] combine_rows_per_src=[5398, 6744, 8188, 7992, 8188, 8192, 8180, 7938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61488 payload_rows_remote=53296 dup_rows=4048 dup_rows_remote=3126 dedup_ratio=0.061768 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4096, 3740, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 15, 0, 3914, 0, 0, 0, 28, 0, 35, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 461, 0] payload_rows_per_dst=[7480, 8186, 5338, 7906, 8002, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 2856, 70, 192, 8, 922] combine_rows_per_src=[7480, 8186, 5338, 7906, 8002, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65118 payload_rows_remote=56926 dup_rows=418 dup_rows_remote=232 dedup_ratio=0.006378 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 4096, 4048, 0, 0, 4096, 0, 8, 0, 0, 0, 2, 93, 0, 0, 4088, 6, 0, 0, 1, 0, 3864, 0, 2, 0, 94, 0, 64, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 19, 0] payload_rows_per_dst=[8096, 8192, 8192, 7920, 8192, 8146, 8188, 8192] dup_rows_per_dst=[0, 16, 186, 2, 128, 0, 48, 38] combine_rows_per_src=[8096, 8192, 8192, 7920, 8192, 8146, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=223862784 dispatch_meta_bytes=229376 combine_payload_bytes=223862784 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=224092160 ingress_bytes=249028608 link_bytes=249028608 token_bytes=447954944 weight_movement_bytes=25165824 total_nvlink_bytes=473120768 weight_to_token_ratio=0.056179 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.894819970e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60838 payload_rows_remote=52744 dup_rows=4698 dup_rows_remote=886 dedup_ratio=0.071686 router_tokens_per_expert=[0, 0, 0, 0, 4043, 0, 4026, 3896, 0, 0, 4066, 6, 0, 11, 0, 0, 0, 132, 14, 0, 4073, 0, 0, 0, 680, 0, 3506, 0, 0, 6, 0, 11, 6, 3766, 8, 6, 0, 0, 0, 65, 0, 0, 0, 4083, 0, 0, 37, 3, 0, 7, 19, 0, 12, 0, 23, 118, 8, 3, 0, 0, 0, 0, 134, 0] payload_rows_per_dst=[8094, 8154, 8174, 8162, 6828, 8180, 7642, 5604] dup_rows_per_dst=[3812, 12, 440, 244, 8, 66, 116, 0] combine_rows_per_src=[8094, 8154, 8174, 8162, 6828, 8180, 7642, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57440 payload_rows_remote=50302 dup_rows=8096 dup_rows_remote=8088 dedup_ratio=0.123535 router_tokens_per_expert=[0, 1830, 0, 3301, 4, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4079, 3, 1, 1, 1, 0, 2851, 0, 0, 14, 0, 0, 0, 0, 0, 3, 0, 0, 6, 2, 0, 0, 0, 0, 6, 0, 3567, 0, 0, 4095, 0, 34, 0, 0, 0, 0, 0, 0, 0, 867, 1, 760, 0, 3371, 0] payload_rows_per_dst=[3668, 8182, 8160, 5730, 8192, 7138, 8190, 8180] dup_rows_per_dst=[0, 6180, 10, 0, 12, 8, 68, 1818] combine_rows_per_src=[3668, 8182, 8160, 5730, 8192, 7138, 8190, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233480192 dispatch_meta_bytes=229368 combine_payload_bytes=233480192 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=233709560 ingress_bytes=334143488 link_bytes=334143488 token_bytes=467189752 weight_movement_bytes=100663296 total_nvlink_bytes=567853048 weight_to_token_ratio=0.215466 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.542445862e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=216383488 dispatch_meta_bytes=230160 combine_payload_bytes=216383488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=216613648 ingress_bytes=241549312 link_bytes=241549312 token_bytes=432997136 weight_movement_bytes=25165824 total_nvlink_bytes=458162960 weight_to_token_ratio=0.058120 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.837911170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=218300416 dispatch_meta_bytes=225688 combine_payload_bytes=218300416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=218526104 ingress_bytes=243466240 link_bytes=243466240 token_bytes=436826520 weight_movement_bytes=25165824 total_nvlink_bytes=461992344 weight_to_token_ratio=0.057611 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.852496777e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233168896 dispatch_meta_bytes=228632 combine_payload_bytes=233168896 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=233397528 ingress_bytes=283500544 link_bytes=283500544 token_bytes=466566424 weight_movement_bytes=50331648 total_nvlink_bytes=516898072 weight_to_token_ratio=0.107877 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.157111573e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=216039424 dispatch_meta_bytes=214520 combine_payload_bytes=216039424 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=216253944 ingress_bytes=291536896 link_bytes=291536896 token_bytes=432293368 weight_movement_bytes=75497472 total_nvlink_bytes=507790840 weight_to_token_ratio=0.174644 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=30720 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.218258926e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=206036992 dispatch_meta_bytes=233560 combine_payload_bytes=206036992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206270552 ingress_bytes=231202816 link_bytes=231202816 token_bytes=412307544 weight_movement_bytes=25165824 total_nvlink_bytes=437473368 weight_to_token_ratio=0.061037 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.759186290e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64208 payload_rows_remote=56016 dup_rows=1328 dup_rows_remote=1328 dedup_ratio=0.020264 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4091, 3900, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 134, 0, 0, 4096, 10, 0, 0, 0, 0, 3661, 0, 0, 0, 0, 0, 416, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0] payload_rows_per_dst=[7800, 8192, 8192, 7322, 8192, 8126, 8192, 8192] dup_rows_per_dst=[0, 6, 288, 0, 832, 0, 0, 202] combine_rows_per_src=[7800, 8192, 8192, 7322, 8192, 8126, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64288 payload_rows_remote=56530 dup_rows=1248 dup_rows_remote=1248 dedup_ratio=0.019043 router_tokens_per_expert=[0, 905, 0, 3155, 0, 0, 0, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 750, 0, 124, 0, 3590, 0] payload_rows_per_dst=[7656, 7758, 8192, 7948, 8192, 8192, 8192, 8158] dup_rows_per_dst=[464, 0, 0, 0, 0, 12, 0, 772] combine_rows_per_src=[7656, 7758, 8192, 7948, 8192, 8192, 8192, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60494 payload_rows_remote=52302 dup_rows=5042 dup_rows_remote=5032 dedup_ratio=0.076935 router_tokens_per_expert=[0, 2147, 0, 1043, 0, 0, 0, 0, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 40, 0, 0, 0, 0, 4011, 0, 0, 3, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 21, 1, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1509, 0, 473, 0, 3799, 0] payload_rows_per_dst=[4294, 7212, 8192, 8028, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 1538, 80, 0, 10, 42, 0, 3372] combine_rows_per_src=[4294, 7212, 8192, 8028, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65440 payload_rows_remote=57276 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.001465 router_tokens_per_expert=[1, 0, 0, 30, 0, 3, 2, 37, 0, 0, 0, 0, 0, 3, 2, 0, 3, 0, 0, 1, 0, 0, 0, 4093, 1, 0, 4052, 4079, 0, 0, 0, 2, 0, 4087, 0, 0, 1, 14, 0, 2, 0, 5, 0, 4083, 0, 2, 0, 0, 0, 4094, 4092, 0, 3, 2, 0, 4073, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8164, 8190, 8174, 8154, 8184] dup_rows_per_dst=[58, 6, 4, 0, 18, 6, 2, 2] combine_rows_per_src=[8192, 8192, 8190, 8164, 8190, 8174, 8154, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=229441536 dispatch_meta_bytes=229376 combine_payload_bytes=229441536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229670912 ingress_bytes=254607360 link_bytes=254607360 token_bytes=459112448 weight_movement_bytes=25165824 total_nvlink_bytes=484278272 weight_to_token_ratio=0.054814 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.937267827e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=231546880 dispatch_meta_bytes=231112 combine_payload_bytes=231546880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=231777992 ingress_bytes=256712704 link_bytes=256712704 token_bytes=463324872 weight_movement_bytes=25165824 total_nvlink_bytes=488490696 weight_to_token_ratio=0.054316 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.953287062e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63196 payload_rows_remote=56538 dup_rows=2340 dup_rows_remote=2136 dedup_ratio=0.035706 router_tokens_per_expert=[0, 3779, 0, 1515, 0, 0, 0, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 24, 0, 2432, 0] payload_rows_per_dst=[7558, 8116, 8158, 8192, 8192, 8188, 8134, 6658] dup_rows_per_dst=[0, 2050, 20, 0, 40, 26, 0, 204] combine_rows_per_src=[7558, 8116, 8158, 8192, 8192, 8188, 8134, 6658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62108 payload_rows_remote=53932 dup_rows=3428 dup_rows_remote=2522 dedup_ratio=0.052307 router_tokens_per_expert=[0, 3374, 0, 1308, 0, 0, 0, 0, 0, 2842, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 453, 0, 0, 0, 0, 4079, 0, 0, 4, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 4017, 0, 0, 4095, 0, 10, 0, 0, 0, 7, 0, 0, 0, 745, 0, 20, 0, 3472, 0] payload_rows_per_dst=[7724, 5684, 8176, 8166, 8192, 8152, 8192, 7822] dup_rows_per_dst=[1640, 0, 906, 2, 16, 180, 32, 652] combine_rows_per_src=[7724, 5684, 8176, 8166, 8192, 8152, 8192, 7822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65136 payload_rows_remote=57128 dup_rows=400 dup_rows_remote=350 dedup_ratio=0.006104 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 101, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 4089, 0, 0, 4022, 4083, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4002, 0, 0, 0, 11, 72, 4095, 4073, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8048, 8192, 8192, 8168, 8190, 8008, 8188, 8150] dup_rows_per_dst=[202, 2, 2, 0, 0, 50, 144, 0] combine_rows_per_src=[8048, 8192, 8192, 8168, 8190, 8008, 8188, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47802 payload_rows_remote=39610 dup_rows=17734 dup_rows_remote=6190 dedup_ratio=0.270599 router_tokens_per_expert=[0, 2907, 9, 4040, 0, 29, 0, 0, 15, 321, 4, 0, 6, 0, 0, 17, 0, 4054, 0, 3993, 1222, 0, 317, 0, 6, 3955, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 34, 0, 3701, 13, 21, 4073, 0, 0, 0, 0, 0, 0, 8, 0, 149, 19, 6, 3, 0, 3815, 6] payload_rows_per_dst=[8192, 676, 4480, 7932, 8110, 7502, 8162, 2748] dup_rows_per_dst=[11544, 50, 960, 4214, 0, 950, 0, 16] combine_rows_per_src=[8192, 676, 4480, 7932, 8110, 7502, 8162, 2748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=214228992 dispatch_meta_bytes=229336 combine_payload_bytes=214228992 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=214458328 ingress_bytes=264560640 link_bytes=264560640 token_bytes=428687320 weight_movement_bytes=50331648 total_nvlink_bytes=479018968 weight_to_token_ratio=0.117409 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.013000787e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234602496 dispatch_meta_bytes=229488 combine_payload_bytes=234602496 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=234831984 ingress_bytes=310099968 link_bytes=310099968 token_bytes=469434480 weight_movement_bytes=75497472 total_nvlink_bytes=544931952 weight_to_token_ratio=0.160826 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.359502456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=231579648 dispatch_meta_bytes=234696 combine_payload_bytes=231579648 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=231814344 ingress_bytes=332242944 link_bytes=332242944 token_bytes=463393992 weight_movement_bytes=100663296 total_nvlink_bytes=564057288 weight_to_token_ratio=0.217230 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.527984918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=220905472 dispatch_meta_bytes=225816 combine_payload_bytes=220905472 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=221131288 ingress_bytes=321568768 link_bytes=321568768 token_bytes=442036760 weight_movement_bytes=100663296 total_nvlink_bytes=542700056 weight_to_token_ratio=0.227726 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.446766772e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233996288 dispatch_meta_bytes=229912 combine_payload_bytes=233996288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234226200 ingress_bytes=259162112 link_bytes=259162112 token_bytes=468222488 weight_movement_bytes=25165824 total_nvlink_bytes=493388312 weight_to_token_ratio=0.053748 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.971924227e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=162242560 dispatch_meta_bytes=183200 combine_payload_bytes=162242560 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=162425760 ingress_bytes=212574208 link_bytes=212574208 token_bytes=324668320 weight_movement_bytes=50331648 total_nvlink_bytes=374999968 weight_to_token_ratio=0.155025 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.617444107e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64870 payload_rows_remote=56678 dup_rows=666 dup_rows_remote=666 dedup_ratio=0.010162 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 12, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4036, 3950, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3972, 0, 0, 0, 0, 322, 4096, 4095, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8074, 8192, 8192, 7900, 8186, 7944, 8192, 8190] dup_rows_per_dst=[58, 0, 0, 0, 0, 8, 600, 0] combine_rows_per_src=[8074, 8192, 8192, 7900, 8186, 7944, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62508 payload_rows_remote=54316 dup_rows=3028 dup_rows_remote=3028 dedup_ratio=0.046204 router_tokens_per_expert=[4, 3072, 0, 4018, 0, 0, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 683, 0, 0, 0, 0, 3663, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 13, 0, 3961, 0] payload_rows_per_dst=[6152, 8132, 8192, 7328, 8192, 8188, 8192, 8132] dup_rows_per_dst=[0, 582, 1366, 0, 2, 508, 0, 570] combine_rows_per_src=[6152, 8132, 8192, 7328, 8192, 8188, 8192, 8132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=232153088 dispatch_meta_bytes=229376 combine_payload_bytes=232153088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=232382464 ingress_bytes=257318912 link_bytes=257318912 token_bytes=464535552 weight_movement_bytes=25165824 total_nvlink_bytes=489701376 weight_to_token_ratio=0.054174 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.957899605e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65236 payload_rows_remote=57256 dup_rows=300 dup_rows_remote=298 dedup_ratio=0.004578 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 4085, 1, 0, 4093, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 3, 0, 1, 0, 1, 0, 0, 0, 2, 0, 3988, 4093, 0, 0, 3, 3655, 524, 0, 2, 0, 0, 2, 0, 0, 4093, 51, 0, 0, 0, 0, 0, 0, 0, 4067, 2, 2, 3, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[8172, 8186, 8192, 7980, 8186, 8190, 8138, 8192] dup_rows_per_dst=[0, 4, 2, 2, 182, 102, 0, 8] combine_rows_per_src=[8172, 8186, 8192, 7980, 8186, 8190, 8138, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64268 payload_rows_remote=56180 dup_rows=1268 dup_rows_remote=1268 dedup_ratio=0.019348 router_tokens_per_expert=[0, 0, 0, 35, 0, 0, 594, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4088, 3972, 1, 1, 0, 0, 0, 4042, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3638, 0, 0, 0, 0, 23, 4095, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 7946, 8088, 7276, 8192, 8192] dup_rows_per_dst=[1244, 0, 2, 2, 0, 0, 20, 0] combine_rows_per_src=[8192, 8192, 8190, 7946, 8088, 7276, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=222478336 dispatch_meta_bytes=229376 combine_payload_bytes=222478336 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=222707712 ingress_bytes=247644160 link_bytes=247644160 token_bytes=445186048 weight_movement_bytes=25165824 total_nvlink_bytes=470351872 weight_to_token_ratio=0.056529 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.884285921e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61434 payload_rows_remote=53242 dup_rows=4102 dup_rows_remote=4098 dedup_ratio=0.062592 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3847, 0, 0, 0, 0, 277, 4096, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5046, 8140, 8182, 7796, 8182, 7704, 8192, 8192] dup_rows_per_dst=[8, 0, 12, 0, 0, 3462, 616, 4] combine_rows_per_src=[5046, 8140, 8182, 7796, 8182, 7704, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65386 payload_rows_remote=57200 dup_rows=150 dup_rows_remote=128 dedup_ratio=0.002289 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 57, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4092, 0, 0, 4076, 4043, 3, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4, 4093, 4096, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8162, 8186, 8186, 8086, 8192, 8190, 8192, 8192] dup_rows_per_dst=[108, 0, 22, 12, 0, 2, 4, 2] combine_rows_per_src=[8162, 8186, 8186, 8086, 8192, 8190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63624 payload_rows_remote=55432 dup_rows=1912 dup_rows_remote=994 dedup_ratio=0.029175 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 3817, 0, 398, 0, 0, 0, 4032, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 3315, 4093, 0, 0, 0, 4095, 227, 0, 1, 0, 1, 0, 448, 0, 4091, 14, 1, 0, 31, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 7966, 8066, 6638, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 478, 0, 0, 454, 918, 60, 2] combine_rows_per_src=[8190, 7966, 8066, 6638, 8192, 8192, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63628 payload_rows_remote=55446 dup_rows=1908 dup_rows_remote=1460 dedup_ratio=0.029114 router_tokens_per_expert=[9, 4, 0, 0, 0, 16, 218, 1, 0, 0, 0, 0, 25, 18, 0, 0, 0, 0, 2, 0, 0, 9, 0, 4074, 5, 3, 4067, 4043, 0, 0, 17, 14, 24, 4021, 0, 1, 13, 29, 6, 18, 4, 0, 0, 3866, 0, 0, 0, 0, 0, 4078, 4076, 0, 0, 0, 0, 4055, 22, 0, 0, 0, 1, 14, 15, 0] payload_rows_per_dst=[8182, 8192, 8152, 8146, 8148, 7740, 6888, 8180] dup_rows_per_dst=[448, 50, 18, 1188, 148, 0, 0, 56] combine_rows_per_src=[8182, 8192, 8152, 8146, 8148, 7740, 6888, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234520576 dispatch_meta_bytes=230216 combine_payload_bytes=234520576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234750792 ingress_bytes=259686400 link_bytes=259686400 token_bytes=469271368 weight_movement_bytes=25165824 total_nvlink_bytes=494437192 weight_to_token_ratio=0.053627 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975913453e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=230113280 dispatch_meta_bytes=229792 combine_payload_bytes=230113280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230343072 ingress_bytes=255279104 link_bytes=255279104 token_bytes=460456352 weight_movement_bytes=25165824 total_nvlink_bytes=485622176 weight_to_token_ratio=0.054654 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.942379023e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=218079232 dispatch_meta_bytes=229360 combine_payload_bytes=218079232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=218308592 ingress_bytes=243245056 link_bytes=243245056 token_bytes=436387824 weight_movement_bytes=25165824 total_nvlink_bytes=461553648 weight_to_token_ratio=0.057668 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.850813822e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=234291200 dispatch_meta_bytes=229312 combine_payload_bytes=234291200 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=234520512 ingress_bytes=259457024 link_bytes=259457024 token_bytes=468811712 weight_movement_bytes=25165824 total_nvlink_bytes=493977536 weight_to_token_ratio=0.053680 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.974168167e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62958 payload_rows_remote=54912 dup_rows=2578 dup_rows_remote=2578 dedup_ratio=0.039337 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4023, 0, 0, 0, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3077, 4096, 0, 0, 0, 3303, 1512, 0, 0, 0, 0, 0, 378, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8046, 8190, 6948, 7018, 8192, 8180, 8192] dup_rows_per_dst=[0, 0, 4, 660, 1158, 756, 0, 0] combine_rows_per_src=[8192, 8046, 8190, 6948, 7018, 8192, 8180, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=227049472 dispatch_meta_bytes=225704 combine_payload_bytes=227049472 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=227275176 ingress_bytes=277381120 link_bytes=277381120 token_bytes=454324648 weight_movement_bytes=50331648 total_nvlink_bytes=504656296 weight_to_token_ratio=0.110783 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.110549827e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=227106816 dispatch_meta_bytes=227624 combine_payload_bytes=227106816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=227334440 ingress_bytes=252272640 link_bytes=252272640 token_bytes=454441256 weight_movement_bytes=25165824 total_nvlink_bytes=479607080 weight_to_token_ratio=0.055378 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.919503305e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64188 payload_rows_remote=56040 dup_rows=1348 dup_rows_remote=1344 dedup_ratio=0.020569 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 647, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 4061, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3480, 0, 0, 6, 0, 4, 4096, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8126, 8192, 6960, 8148, 8192] dup_rows_per_dst=[1328, 0, 0, 2, 2, 12, 4, 0] combine_rows_per_src=[8186, 8192, 8192, 8126, 8192, 6960, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=224919552 dispatch_meta_bytes=229960 combine_payload_bytes=224919552 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=225149512 ingress_bytes=300417024 link_bytes=300417024 token_bytes=450069064 weight_movement_bytes=75497472 total_nvlink_bytes=525566536 weight_to_token_ratio=0.167746 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.285826440e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65268 payload_rows_remote=57080 dup_rows=268 dup_rows_remote=260 dedup_ratio=0.004089 router_tokens_per_expert=[0, 4076, 7, 0, 0, 0, 1, 4, 4091, 0, 5, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 3, 2, 0, 0, 0, 2, 0, 0, 0, 1, 0, 4093, 3, 0, 1, 0, 0, 0, 4092, 0, 3, 93, 0, 0, 0, 0, 29, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 4087, 0] payload_rows_per_dst=[8158, 8178, 8120, 8188, 8192, 8192, 8188, 8052] dup_rows_per_dst=[18, 0, 0, 8, 4, 184, 54, 0] combine_rows_per_src=[8158, 8178, 8120, 8188, 8192, 8192, 8188, 8052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62774 payload_rows_remote=54590 dup_rows=2762 dup_rows_remote=126 dedup_ratio=0.042145 router_tokens_per_expert=[0, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 0, 3050, 0, 7, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 4096, 0, 0, 0, 3091, 2319, 0, 0, 0, 0, 0, 60, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6110, 8188, 7524, 8184, 8192, 8192, 8192] dup_rows_per_dst=[4, 4, 0, 0, 2636, 118, 0, 0] combine_rows_per_src=[8192, 6110, 8188, 7524, 8184, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=229539840 dispatch_meta_bytes=229536 combine_payload_bytes=229539840 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=229769376 ingress_bytes=355368960 link_bytes=355368960 token_bytes=459309216 weight_movement_bytes=125829120 total_nvlink_bytes=585138336 weight_to_token_ratio=0.273953 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.703947179e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65032 payload_rows_remote=56846 dup_rows=504 dup_rows_remote=492 dedup_ratio=0.007690 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4088, 5, 0, 3884, 0, 0, 0, 0, 0, 4093, 0, 6, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4066, 4096, 1, 0, 0, 4085, 7, 0, 0, 0, 0, 0, 243, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 7770, 8186, 8138, 8186, 8192, 8192, 8192] dup_rows_per_dst=[0, 8, 12, 0, 0, 484, 0, 0] combine_rows_per_src=[8176, 7770, 8186, 8138, 8186, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63192 payload_rows_remote=55000 dup_rows=2344 dup_rows_remote=2344 dedup_ratio=0.035767 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 3671, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 4096, 0, 0, 0, 3197, 1549, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7342, 8192, 7718, 7172, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 22, 0, 1254, 1048, 20, 0] combine_rows_per_src=[8192, 7342, 8192, 7718, 7172, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=233799680 dispatch_meta_bytes=229360 combine_payload_bytes=233799680 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=234029040 ingress_bytes=284131328 link_bytes=284131328 token_bytes=467828720 weight_movement_bytes=50331648 total_nvlink_bytes=518160368 weight_to_token_ratio=0.107586 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.161911110e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63700 payload_rows_remote=55508 dup_rows=1836 dup_rows_remote=396 dedup_ratio=0.028015 router_tokens_per_expert=[0, 3400, 1, 0, 0, 0, 1, 0, 4094, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 2, 11, 0, 0, 3958, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3999, 0, 0, 1, 0, 0, 0, 4095, 720, 0, 0, 0, 1, 0, 0, 181, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0] payload_rows_per_dst=[6804, 8192, 7940, 8190, 8000, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 2, 0, 1440, 392, 0] combine_rows_per_src=[6804, 8192, 7940, 8190, 8000, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58062 payload_rows_remote=49964 dup_rows=7474 dup_rows_remote=7354 dedup_ratio=0.114044 router_tokens_per_expert=[0, 0, 0, 0, 92, 0, 0, 1, 3921, 0, 6, 3956, 0, 80, 0, 0, 0, 4082, 0, 0, 44, 19, 0, 0, 0, 0, 0, 0, 1, 16, 3, 4072, 4072, 0, 0, 23, 1777, 2321, 0, 16, 0, 0, 6, 21, 0, 4080, 7, 20, 0, 9, 1, 0, 0, 0, 4076, 4, 16, 0, 0, 11, 0, 0, 11, 4] payload_rows_per_dst=[8098, 8130, 8180, 8186, 4684, 4472, 8162, 8150] dup_rows_per_dst=[120, 3546, 110, 3552, 72, 14, 18, 42] combine_rows_per_src=[8098, 8130, 8180, 8186, 4684, 4472, 8162, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=223600640 dispatch_meta_bytes=218864 combine_payload_bytes=223600640 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=223819504 ingress_bytes=324263936 link_bytes=324263936 token_bytes=447420144 weight_movement_bytes=100663296 total_nvlink_bytes=548083440 weight_to_token_ratio=0.224986 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=23808 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.467273887e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232841216 dispatch_meta_bytes=229352 combine_payload_bytes=232841216 prefetch_bytes=0 reduce_bytes=0 egress_bytes=233070568 ingress_bytes=232841216 link_bytes=233070568 token_bytes=465911784 weight_movement_bytes=0 total_nvlink_bytes=465911784 weight_to_token_ratio=0.000000 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.773397724e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=225280000 dispatch_meta_bytes=229376 combine_payload_bytes=225280000 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=225509376 ingress_bytes=250445824 link_bytes=250445824 token_bytes=450789376 weight_movement_bytes=25165824 total_nvlink_bytes=475955200 weight_to_token_ratio=0.055826 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.905603346e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60634 payload_rows_remote=52442 dup_rows=4902 dup_rows_remote=4902 dedup_ratio=0.074799 router_tokens_per_expert=[0, 2749, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 1328, 0, 0, 0, 0, 0, 0, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8174, 8192, 8182, 8192, 8192, 3318, 8192, 8192] dup_rows_per_dst=[4622, 0, 0, 0, 218, 10, 52, 0] combine_rows_per_src=[8174, 8192, 8182, 8192, 8192, 3318, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6880052256 prefetch_bytes=427819008 reduce_bytes=377487360 weight_movement_bytes=805306368 egress_bytes=3441878048 ingress_bytes=4243480576 link_bytes=4243480576 total_nvlink_bytes=7685358624 weight_to_token_ratio=0.117049 prefetch_share_of_ingress=0.100818 dedup_ratio=0.089317 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=15 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.037914 gemm_inner_pad_upper_bound=17653 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.017999007e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=227360768 dispatch_meta_bytes=223616 combine_payload_bytes=227360768 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=227584384 ingress_bytes=277692416 link_bytes=277692416 token_bytes=454945152 weight_movement_bytes=50331648 total_nvlink_bytes=505276800 weight_to_token_ratio=0.110632 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=24960 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.112918430e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=204652544 dispatch_meta_bytes=229272 combine_payload_bytes=204652544 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=204881816 ingress_bytes=254984192 link_bytes=254984192 token_bytes=409534360 weight_movement_bytes=50331648 total_nvlink_bytes=459866008 weight_to_token_ratio=0.122900 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.940135083e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65228 payload_rows_remote=57038 dup_rows=308 dup_rows_remote=308 dedup_ratio=0.004700 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 0, 4096, 5, 0, 3968, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 3575, 504, 0, 0, 0, 0, 0, 140, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7946, 8192, 8190, 8140, 8192, 8190, 8186] dup_rows_per_dst=[8, 0, 0, 0, 18, 282, 0, 0] combine_rows_per_src=[8192, 7946, 8192, 8190, 8140, 8192, 8190, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=214802432 dispatch_meta_bytes=229376 combine_payload_bytes=214802432 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=215031808 ingress_bytes=290299904 link_bytes=290299904 token_bytes=429834240 weight_movement_bytes=75497472 total_nvlink_bytes=505331712 weight_to_token_ratio=0.175643 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.208846846e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6603380632 prefetch_bytes=327155712 reduce_bytes=377487360 weight_movement_bytes=704643072 egress_bytes=3303495576 ingress_bytes=4004528128 link_bytes=4004528128 total_nvlink_bytes=7308023704 weight_to_token_ratio=0.106709 prefetch_share_of_ingress=0.081696 dedup_ratio=0.112322 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=15 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.038603 gemm_inner_pad_upper_bound=16637 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.904364505e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65472 payload_rows_remote=57280 dup_rows=64 dup_rows_remote=62 dedup_ratio=0.000977 router_tokens_per_expert=[0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4095, 17, 0, 13, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0] payload_rows_per_dst=[8180, 8192, 8142, 8192, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 0, 0, 2, 58, 4, 0] combine_rows_per_src=[8180, 8192, 8142, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65126 payload_rows_remote=56934 dup_rows=410 dup_rows_remote=364 dedup_ratio=0.006256 router_tokens_per_expert=[0, 3930, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 22, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4075, 139, 0, 14, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4096, 1] payload_rows_per_dst=[7860, 8192, 8192, 8192, 8188, 8176, 8192, 8134] dup_rows_per_dst=[0, 0, 46, 0, 0, 280, 84, 0] combine_rows_per_src=[7860, 8192, 8192, 8192, 8188, 8176, 8192, 8134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233627648 dispatch_meta_bytes=229384 combine_payload_bytes=233627648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233857032 ingress_bytes=258793472 link_bytes=258793472 token_bytes=467484680 weight_movement_bytes=25165824 total_nvlink_bytes=492650504 weight_to_token_ratio=0.053832 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.969119303e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6594332792 prefetch_bytes=352321536 reduce_bytes=352321536 weight_movement_bytes=704643072 egress_bytes=3298986104 ingress_bytes=3999989760 link_bytes=3999989760 total_nvlink_bytes=7298975864 weight_to_token_ratio=0.106856 prefetch_share_of_ingress=0.088081 dedup_ratio=0.113409 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=14 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=16129 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.902206271e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57906 payload_rows_remote=49714 dup_rows=7630 dup_rows_remote=6700 dedup_ratio=0.116425 router_tokens_per_expert=[0, 753, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[1506, 8178, 8192, 8192, 8192, 8192, 7262, 8192] dup_rows_per_dst=[0, 0, 0, 14, 0, 6686, 0, 930] combine_rows_per_src=[1506, 8178, 8192, 8192, 8192, 8192, 7262, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56498 payload_rows_remote=48322 dup_rows=9038 dup_rows_remote=976 dedup_ratio=0.137909 router_tokens_per_expert=[0, 4025, 11, 1, 0, 0, 13, 0, 4066, 0, 1, 34, 0, 2, 4068, 0, 0, 0, 0, 0, 0, 7, 3969, 0, 0, 0, 16, 0, 0, 5, 0, 0, 0, 0, 0, 4027, 26, 6, 24, 0, 18, 0, 4069, 39, 16, 0, 0, 14, 9, 11, 99, 0, 0, 4, 35, 0, 0, 50, 0, 0, 0, 4029, 4074, 0] payload_rows_per_dst=[8176, 8170, 8068, 8164, 8110, 146, 8192, 7472] dup_rows_per_dst=[8062, 50, 560, 10, 56, 28, 272, 0] combine_rows_per_src=[8176, 8170, 8068, 8164, 8110, 146, 8192, 7472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=234618880 dispatch_meta_bytes=229368 combine_payload_bytes=234618880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234848248 ingress_bytes=259784704 link_bytes=259784704 token_bytes=469467128 weight_movement_bytes=25165824 total_nvlink_bytes=494632952 weight_to_token_ratio=0.053605 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.976661433e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=233201664 dispatch_meta_bytes=229192 combine_payload_bytes=233201664 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233430856 ingress_bytes=258367488 link_bytes=258367488 token_bytes=466632520 weight_movement_bytes=25165824 total_nvlink_bytes=491798344 weight_to_token_ratio=0.053931 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.965878057e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=203628544 dispatch_meta_bytes=225656 combine_payload_bytes=203628544 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=203854200 ingress_bytes=279126016 link_bytes=279126016 token_bytes=407482744 weight_movement_bytes=75497472 total_nvlink_bytes=482980216 weight_to_token_ratio=0.185278 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.123826469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=197926912 dispatch_meta_bytes=197192 combine_payload_bytes=197926912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198124104 ingress_bytes=223092736 link_bytes=223092736 token_bytes=396051016 weight_movement_bytes=25165824 total_nvlink_bytes=421216840 weight_to_token_ratio=0.063542 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.697477952e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6763432200 prefetch_bytes=427819008 reduce_bytes=503316480 weight_movement_bytes=931135488 egress_bytes=3383552264 ingress_bytes=4311015424 link_bytes=4311015424 total_nvlink_bytes=7694567688 weight_to_token_ratio=0.137672 prefetch_share_of_ingress=0.099239 dedup_ratio=0.100616 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=20 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039062 gemm_inner_pad_upper_bound=17018 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.050115392e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6708777968 prefetch_bytes=301989888 reduce_bytes=377487360 weight_movement_bytes=679477248 egress_bytes=3356210160 ingress_bytes=4032045056 link_bytes=4032274408 total_nvlink_bytes=7388255216 weight_to_token_ratio=0.101282 prefetch_share_of_ingress=0.074897 dedup_ratio=0.103788 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=15 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039982 gemm_inner_pad_upper_bound=16510 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.917559326e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61210 payload_rows_remote=53018 dup_rows=4326 dup_rows_remote=4250 dedup_ratio=0.066010 router_tokens_per_expert=[0, 1988, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4074, 2108, 0, 39, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0] payload_rows_per_dst=[3976, 8192, 8092, 8192, 8190, 8192, 8192, 8184] dup_rows_per_dst=[0, 0, 0, 0, 0, 4250, 76, 0] combine_rows_per_src=[3976, 8192, 8092, 8192, 8190, 8192, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6201340400 prefetch_bytes=402653184 reduce_bytes=377487360 weight_movement_bytes=780140544 egress_bytes=3102470640 ingress_bytes=3879010304 link_bytes=3879010304 total_nvlink_bytes=6981480944 weight_to_token_ratio=0.125802 prefetch_share_of_ingress=0.103803 dedup_ratio=0.161076 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=15 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039062 gemm_inner_pad_upper_bound=16510 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.844674154e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6015649600 prefetch_bytes=402653184 reduce_bytes=226492416 weight_movement_bytes=629145600 egress_bytes=3009521472 ingress_bytes=3635273728 link_bytes=3635273728 total_nvlink_bytes=6644795200 weight_to_token_ratio=0.104585 prefetch_share_of_ingress=0.110763 dedup_ratio=0.178356 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=9 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039292 gemm_inner_pad_upper_bound=17018 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.728764546e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=217161728 dispatch_meta_bytes=229072 combine_payload_bytes=217161728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=217390800 ingress_bytes=242327552 link_bytes=242327552 token_bytes=434552528 weight_movement_bytes=25165824 total_nvlink_bytes=459718352 weight_to_token_ratio=0.057912 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.843832677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6657495984 prefetch_bytes=402653184 reduce_bytes=452984832 weight_movement_bytes=855638016 egress_bytes=3330560944 ingress_bytes=4182573056 link_bytes=4182573056 total_nvlink_bytes=7513134000 weight_to_token_ratio=0.128522 prefetch_share_of_ingress=0.096269 dedup_ratio=0.105715 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=18 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.038833 gemm_inner_pad_upper_bound=16891 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.989034266e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-09-21 08:31:46] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 219.5 | throughput per GPU (TFLOP/s/GPU): 146.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.710195E+00 | loss scale: 1.0 | grad norm: 3.798 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45634 payload_rows_remote=39934 dup_rows=19902 dup_rows_remote=17668 dedup_ratio=0.303680 router_tokens_per_expert=[123, 242, 933, 1, 1, 478, 792, 411, 1, 589, 36, 1, 58, 4, 545, 84, 2, 1170, 581, 1745, 35, 307, 202, 0, 42, 22, 26, 409, 888, 1114, 1205, 1738, 790, 537, 498, 1164, 147, 1348, 421, 728, 1634, 875, 1482, 70, 587, 732, 21, 48, 1188, 529, 689, 49, 159, 1386, 1700, 46, 1114, 31, 411, 2, 16, 13, 548, 20] payload_rows_per_dst=[4548, 5438, 5968, 6870, 4858, 5700, 6680, 5572] dup_rows_per_dst=[1414, 2222, 2116, 4018, 1384, 2234, 4812, 1702] combine_rows_per_src=[4548, 5438, 5968, 6870, 4858, 5700, 6680, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46142 payload_rows_remote=39656 dup_rows=19394 dup_rows_remote=16076 dedup_ratio=0.295929 router_tokens_per_expert=[43, 278, 964, 1, 0, 933, 774, 248, 0, 975, 36, 2, 12, 57, 544, 41, 1, 826, 931, 1461, 59, 165, 270, 4, 94, 30, 103, 221, 1361, 374, 1115, 1894, 666, 359, 395, 1212, 276, 932, 829, 1077, 1837, 348, 1920, 52, 563, 842, 18, 30, 755, 743, 748, 22, 159, 1196, 1654, 33, 1384, 58, 297, 2, 7, 42, 483, 12] payload_rows_per_dst=[5220, 5900, 5666, 6486, 6162, 5584, 5150, 5974] dup_rows_per_dst=[1842, 2804, 1768, 3318, 3268, 1796, 2162, 2436] combine_rows_per_src=[5220, 5900, 5666, 6486, 6162, 5584, 5150, 5974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45900 payload_rows_remote=39840 dup_rows=19636 dup_rows_remote=16512 dedup_ratio=0.299622 router_tokens_per_expert=[80, 588, 1245, 1, 1, 823, 1303, 343, 0, 332, 126, 7, 14, 215, 681, 192, 25, 736, 1053, 1344, 63, 269, 299, 31, 150, 138, 105, 222, 995, 561, 848, 1139, 1102, 416, 446, 1134, 234, 1341, 457, 550, 1393, 893, 1617, 164, 632, 817, 29, 105, 901, 617, 777, 130, 285, 592, 1727, 30, 1291, 56, 324, 27, 28, 16, 691, 17] payload_rows_per_dst=[6732, 6060, 6142, 4770, 6204, 5784, 4318, 5890] dup_rows_per_dst=[4314, 3124, 2682, 1268, 2560, 2282, 1162, 2244] combine_rows_per_src=[6732, 6060, 6142, 4770, 6204, 5784, 4318, 5890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45864 payload_rows_remote=39908 dup_rows=19672 dup_rows_remote=16694 dedup_ratio=0.300171 router_tokens_per_expert=[118, 108, 1195, 0, 0, 654, 761, 451, 3, 458, 50, 1, 7, 34, 233, 117, 3, 1019, 775, 1971, 20, 267, 259, 11, 65, 2, 83, 350, 1095, 624, 1078, 2110, 1267, 409, 565, 971, 174, 781, 524, 1106, 1415, 550, 1843, 73, 685, 564, 5, 25, 1248, 527, 669, 32, 177, 984, 1768, 25, 1017, 13, 178, 3, 7, 9, 1224, 8] payload_rows_per_dst=[4936, 4324, 6140, 6916, 5956, 5158, 6278, 6156] dup_rows_per_dst=[1638, 986, 2510, 3898, 2978, 1476, 3738, 2448] combine_rows_per_src=[4936, 4324, 6140, 6916, 5956, 5158, 6278, 6156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46204 payload_rows_remote=40094 dup_rows=19332 dup_rows_remote=16742 dedup_ratio=0.294983 router_tokens_per_expert=[176, 341, 907, 0, 0, 628, 846, 493, 2, 726, 16, 1, 17, 109, 256, 101, 4, 781, 837, 1763, 30, 123, 199, 3, 105, 7, 83, 270, 858, 381, 1633, 1504, 520, 435, 578, 1598, 132, 1205, 900, 945, 1187, 643, 1516, 57, 1085, 997, 10, 12, 929, 530, 645, 31, 320, 849, 1534, 76, 1435, 11, 250, 6, 4, 6, 1091, 31] payload_rows_per_dst=[5060, 5788, 5658, 5164, 5204, 7040, 6180, 6110] dup_rows_per_dst=[1722, 2274, 1822, 1252, 1816, 4208, 3648, 2590] combine_rows_per_src=[5060, 5788, 5658, 5164, 5204, 7040, 6180, 6110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45936 payload_rows_remote=40622 dup_rows=19600 dup_rows_remote=17852 dedup_ratio=0.299072 router_tokens_per_expert=[114, 257, 1276, 3, 1, 676, 840, 658, 15, 942, 33, 0, 15, 11, 201, 108, 9, 802, 547, 1605, 20, 143, 248, 6, 118, 10, 142, 390, 980, 452, 846, 1582, 622, 503, 543, 1371, 248, 1521, 704, 754, 1566, 594, 1665, 63, 663, 987, 26, 31, 687, 729, 461, 49, 264, 1501, 1946, 14, 1147, 45, 305, 15, 49, 25, 606, 14] payload_rows_per_dst=[6526, 6454, 5314, 4642, 6132, 5748, 5344, 5776] dup_rows_per_dst=[4288, 3488, 1748, 1234, 3000, 2112, 1764, 1966] combine_rows_per_src=[6526, 6454, 5314, 4642, 6132, 5748, 5344, 5776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46632 payload_rows_remote=40030 dup_rows=18904 dup_rows_remote=14330 dedup_ratio=0.288452 router_tokens_per_expert=[90, 119, 1497, 1, 0, 630, 838, 337, 2, 1068, 30, 5, 8, 4, 518, 29, 1, 859, 1178, 1558, 32, 61, 253, 2, 61, 23, 63, 190, 778, 754, 887, 1680, 954, 845, 475, 829, 228, 1185, 455, 811, 1694, 569, 2241, 57, 347, 675, 14, 72, 708, 562, 503, 57, 81, 1577, 2069, 31, 1314, 41, 224, 2, 21, 29, 532, 10] payload_rows_per_dst=[5240, 5438, 5916, 5976, 5538, 5556, 6602, 6366] dup_rows_per_dst=[1784, 1918, 1972, 2382, 1998, 1814, 4574, 2462] combine_rows_per_src=[5240, 5438, 5916, 5976, 5538, 5556, 6602, 6366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 3, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163569664 dispatch_meta_bytes=230408 combine_payload_bytes=163569664 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=163800072 ingress_bytes=213901312 link_bytes=213901312 token_bytes=327369736 weight_movement_bytes=50331648 total_nvlink_bytes=377701384 weight_to_token_ratio=0.153746 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.627541835e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162430976 dispatch_meta_bytes=222928 combine_payload_bytes=162430976 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=162653904 ingress_bytes=237928448 link_bytes=237928448 token_bytes=325084880 weight_movement_bytes=75497472 total_nvlink_bytes=400582352 weight_to_token_ratio=0.232239 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.810360578e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163184640 dispatch_meta_bytes=225408 combine_payload_bytes=163184640 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=163410048 ingress_bytes=238682112 link_bytes=238682112 token_bytes=326594688 weight_movement_bytes=75497472 total_nvlink_bytes=402092160 weight_to_token_ratio=0.231166 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.033088 gemm_active_groups=12 gemm_peak_group_rows=9728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.816095090e-04 prefetch_bytes_per_gemm_flop=5.744485294e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163463168 dispatch_meta_bytes=226408 combine_payload_bytes=163463168 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=163689576 ingress_bytes=264126464 link_bytes=264126464 token_bytes=327152744 weight_movement_bytes=100663296 total_nvlink_bytes=427816040 weight_to_token_ratio=0.307695 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=7552 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.009697210e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=164225024 dispatch_meta_bytes=227344 combine_payload_bytes=164225024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=164452368 ingress_bytes=189390848 link_bytes=189390848 token_bytes=328677392 weight_movement_bytes=25165824 total_nvlink_bytes=353843216 weight_to_token_ratio=0.076567 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.441045524e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166387712 dispatch_meta_bytes=233896 combine_payload_bytes=166387712 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166621608 ingress_bytes=191553536 link_bytes=191553536 token_bytes=333009320 weight_movement_bytes=25165824 total_nvlink_bytes=358175144 weight_to_token_ratio=0.075571 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.457501081e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163962880 dispatch_meta_bytes=217440 combine_payload_bytes=163962880 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=164180320 ingress_bytes=239460352 link_bytes=239460352 token_bytes=328143200 weight_movement_bytes=75497472 total_nvlink_bytes=403640672 weight_to_token_ratio=0.230075 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=7424 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.822016598e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50148 payload_rows_remote=45574 dup_rows=15388 dup_rows_remote=15152 dedup_ratio=0.234802 router_tokens_per_expert=[883, 0, 1, 0, 0, 1389, 5, 1257, 180, 230, 3242, 1940, 2, 0, 1596, 1, 0, 0, 129, 5, 597, 32, 866, 0, 0, 2198, 0, 0, 4, 188, 1337, 15, 0, 0, 0, 0, 2931, 2703, 711, 0, 187, 4, 9, 0, 0, 0, 265, 2, 4, 4, 791, 767, 1178, 0, 5, 0, 476, 6, 2204, 1414, 616, 0, 2394, 0] payload_rows_per_dst=[5844, 6892, 6818, 4574, 6294, 6898, 5270, 7558] dup_rows_per_dst=[1624, 3680, 1846, 236, 592, 520, 1362, 5528] combine_rows_per_src=[5844, 6892, 6818, 4574, 6294, 6898, 5270, 7558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51074 payload_rows_remote=44016 dup_rows=14462 dup_rows_remote=11892 dedup_ratio=0.220673 router_tokens_per_expert=[724, 0, 0, 0, 0, 2232, 37, 620, 9, 507, 2628, 680, 1, 0, 366, 0, 0, 0, 156, 0, 267, 96, 43, 0, 5, 463, 0, 0, 27, 590, 3129, 0, 2, 1, 0, 0, 1788, 4055, 2509, 0, 978, 0, 1208, 0, 0, 0, 0, 0, 0, 0, 103, 3378, 1128, 0, 18, 1, 577, 8, 744, 472, 1856, 0, 1362, 0] payload_rows_per_dst=[5982, 3304, 8112, 6144, 6664, 7058, 7228, 6582] dup_rows_per_dst=[1244, 536, 1122, 1570, 1936, 2570, 2028, 3456] combine_rows_per_src=[5982, 3304, 8112, 6144, 6664, 7058, 7228, 6582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49856 payload_rows_remote=42052 dup_rows=15680 dup_rows_remote=10244 dedup_ratio=0.239258 router_tokens_per_expert=[33, 0, 0, 0, 0, 3882, 1, 696, 12, 115, 3840, 27, 0, 0, 179, 0, 1, 0, 5, 5, 2110, 5, 14, 0, 0, 2606, 0, 1, 4, 1300, 3209, 0, 2, 7, 26, 0, 1805, 1912, 847, 1, 2295, 0, 124, 0, 0, 0, 61, 0, 0, 0, 1814, 293, 159, 0, 13, 0, 933, 0, 424, 1490, 2422, 1, 94, 0] payload_rows_per_dst=[7974, 7804, 7080, 6158, 3618, 5816, 6748, 4658] dup_rows_per_dst=[2944, 5436, 1024, 1664, 64, 668, 2654, 1226] combine_rows_per_src=[7974, 7804, 7080, 6158, 3618, 5816, 6748, 4658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54054 payload_rows_remote=46810 dup_rows=11482 dup_rows_remote=9170 dedup_ratio=0.175201 router_tokens_per_expert=[1181, 0, 10, 0, 0, 2375, 0, 1647, 0, 92, 3790, 118, 0, 0, 883, 0, 0, 0, 195, 3, 76, 97, 60, 7, 0, 634, 0, 2, 0, 3423, 2263, 0, 0, 0, 0, 0, 1720, 3011, 3058, 0, 537, 0, 202, 0, 0, 0, 81, 0, 1, 10, 121, 4020, 750, 0, 2, 3, 453, 0, 882, 707, 310, 0, 44, 0] payload_rows_per_dst=[7426, 5668, 6532, 7268, 7244, 7710, 8092, 4114] dup_rows_per_dst=[3000, 1044, 366, 850, 2312, 1510, 1722, 678] combine_rows_per_src=[7426, 5668, 6532, 7268, 7244, 7710, 8092, 4114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43890 payload_rows_remote=38930 dup_rows=21646 dup_rows_remote=20096 dedup_ratio=0.330292 router_tokens_per_expert=[368, 0, 0, 0, 2, 2317, 2, 919, 6, 1118, 3604, 1354, 1, 1, 296, 3, 0, 0, 113, 113, 3, 134, 68, 0, 0, 2399, 0, 1, 1, 64, 2418, 0, 0, 0, 0, 0, 1290, 3995, 3172, 1, 55, 0, 1147, 0, 1, 0, 1698, 0, 117, 0, 33, 2365, 456, 0, 39, 3, 1112, 9, 286, 956, 625, 0, 101, 2] payload_rows_per_dst=[5648, 4548, 758, 6562, 8154, 7870, 5390, 4960] dup_rows_per_dst=[1568, 1010, 104, 3204, 8434, 5140, 636, 1550] combine_rows_per_src=[5648, 4548, 758, 6562, 8154, 7870, 5390, 4960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49470 payload_rows_remote=41390 dup_rows=16066 dup_rows_remote=12168 dedup_ratio=0.245148 router_tokens_per_expert=[954, 0, 0, 0, 0, 887, 1, 845, 2, 553, 3603, 434, 8, 0, 380, 0, 0, 1, 127, 0, 1616, 222, 41, 0, 1, 1400, 2, 0, 1, 441, 3223, 0, 0, 9, 0, 0, 1688, 3982, 3554, 1, 155, 0, 383, 3, 0, 0, 78, 0, 1, 0, 538, 908, 1713, 0, 0, 0, 246, 0, 1061, 890, 715, 0, 2101, 0] payload_rows_per_dst=[7774, 7054, 8080, 3454, 3396, 7650, 5792, 6270] dup_rows_per_dst=[4708, 2146, 3898, 236, 0, 794, 1958, 2326] combine_rows_per_src=[7774, 7054, 8080, 3454, 3396, 7650, 5792, 6270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48078 payload_rows_remote=42110 dup_rows=17458 dup_rows_remote=16364 dedup_ratio=0.266388 router_tokens_per_expert=[231, 0, 0, 0, 0, 2199, 0, 677, 0, 3, 3731, 623, 1, 0, 578, 0, 0, 0, 578, 6, 1236, 227, 1259, 0, 0, 2427, 1, 0, 119, 1448, 1373, 0, 0, 0, 3, 1, 2238, 2906, 2916, 0, 620, 1, 184, 0, 1, 0, 86, 1, 4, 0, 223, 2731, 563, 0, 10, 0, 71, 3, 1296, 1739, 252, 0, 202, 0] payload_rows_per_dst=[5562, 2178, 6498, 6924, 7504, 7668, 5968, 5776] dup_rows_per_dst=[652, 232, 3600, 3812, 5138, 1580, 1094, 1350] combine_rows_per_src=[5562, 2178, 6498, 6924, 7504, 7668, 5968, 5776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=186671104 dispatch_meta_bytes=242904 combine_payload_bytes=186671104 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=186914008 ingress_bytes=237002752 link_bytes=237002752 token_bytes=373585112 weight_movement_bytes=50331648 total_nvlink_bytes=423916760 weight_to_token_ratio=0.134726 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.803317101e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45586 payload_rows_remote=39124 dup_rows=19950 dup_rows_remote=16140 dedup_ratio=0.304413 router_tokens_per_expert=[212, 405, 649, 0, 0, 829, 958, 429, 1, 575, 18, 2, 7, 67, 258, 132, 1, 728, 709, 1510, 27, 410, 251, 29, 114, 8, 54, 322, 1301, 466, 1040, 1654, 539, 450, 729, 1214, 96, 1321, 722, 1064, 1521, 711, 1693, 28, 1119, 903, 19, 87, 973, 516, 250, 24, 102, 1392, 1703, 63, 1150, 37, 311, 6, 28, 14, 799, 18] payload_rows_per_dst=[6462, 6274, 6558, 5406, 6026, 6204, 2800, 5856] dup_rows_per_dst=[3810, 3044, 3556, 1936, 2452, 2840, 324, 1988] combine_rows_per_src=[6462, 6274, 6558, 5406, 6026, 6204, 2800, 5856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159457280 dispatch_meta_bytes=236104 combine_payload_bytes=159457280 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159693384 ingress_bytes=234954752 link_bytes=234954752 token_bytes=319150664 weight_movement_bytes=75497472 total_nvlink_bytes=394648136 weight_to_token_ratio=0.236557 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787734188e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=180289536 dispatch_meta_bytes=223632 combine_payload_bytes=180289536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=180513168 ingress_bytes=205455360 link_bytes=205455360 token_bytes=360802704 weight_movement_bytes=25165824 total_nvlink_bytes=385968528 weight_to_token_ratio=0.069750 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.563277899e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=172244992 dispatch_meta_bytes=209184 combine_payload_bytes=172244992 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=172454176 ingress_bytes=222576640 link_bytes=222576640 token_bytes=344699168 weight_movement_bytes=50331648 total_nvlink_bytes=395030816 weight_to_token_ratio=0.146016 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.693551057e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=191733760 dispatch_meta_bytes=223920 combine_payload_bytes=191733760 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=191957680 ingress_bytes=267231232 link_bytes=267231232 token_bytes=383691440 weight_movement_bytes=75497472 total_nvlink_bytes=459188912 weight_to_token_ratio=0.196766 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=15488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.033320907e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=169533440 dispatch_meta_bytes=214232 combine_payload_bytes=169533440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=169747672 ingress_bytes=194699264 link_bytes=194699264 token_bytes=339281112 weight_movement_bytes=25165824 total_nvlink_bytes=364446936 weight_to_token_ratio=0.074174 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.481436436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=172482560 dispatch_meta_bytes=233896 combine_payload_bytes=172482560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=172716456 ingress_bytes=197648384 link_bytes=197648384 token_bytes=345199016 weight_movement_bytes=25165824 total_nvlink_bytes=370364840 weight_to_token_ratio=0.072902 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.503875832e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52812 payload_rows_remote=44628 dup_rows=12724 dup_rows_remote=12700 dedup_ratio=0.194153 router_tokens_per_expert=[0, 0, 0, 0, 3806, 0, 753, 0, 3620, 0, 0, 20, 0, 3123, 0, 15, 535, 0, 803, 152, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 0, 8, 1, 1707, 0, 3, 2, 1902, 1246, 0, 0, 0, 0, 0, 2733, 0, 0, 0, 85, 19, 4091, 2307, 0, 1812, 3887, 4, 0, 11, 2, 0, 63, 8, 45, 0] payload_rows_per_dst=[7798, 6272, 2588, 8184, 8102, 5466, 6602, 7800] dup_rows_per_dst=[1320, 44, 392, 24, 8860, 0, 1852, 232] combine_rows_per_src=[7798, 6272, 2588, 8184, 8102, 5466, 6602, 7800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160251904 dispatch_meta_bytes=221056 combine_payload_bytes=160251904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160472960 ingress_bytes=185417728 link_bytes=185417728 token_bytes=320724864 weight_movement_bytes=25165824 total_nvlink_bytes=345890688 weight_to_token_ratio=0.078465 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410814672e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52586 payload_rows_remote=44906 dup_rows=12950 dup_rows_remote=11376 dedup_ratio=0.197601 router_tokens_per_expert=[0, 0, 0, 0, 2234, 0, 3011, 0, 2265, 0, 0, 44, 0, 1971, 0, 75, 2848, 26, 3390, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 19, 399, 0, 0, 351, 1504, 98, 0, 0, 0, 0, 6, 2557, 0, 0, 0, 67, 1, 4045, 3167, 2, 55, 3689, 0, 0, 260, 0, 0, 390, 120, 166, 2] payload_rows_per_dst=[6022, 4156, 7446, 8096, 6384, 6218, 6584, 7680] dup_rows_per_dst=[0, 24, 5084, 4, 2888, 2878, 498, 1574] combine_rows_per_src=[6022, 4156, 7446, 8096, 6384, 6218, 6584, 7680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49810 payload_rows_remote=42116 dup_rows=15726 dup_rows_remote=11242 dedup_ratio=0.239960 router_tokens_per_expert=[0, 6, 0, 18, 3367, 0, 3166, 0, 1327, 0, 16, 2919, 2, 1821, 0, 4, 1205, 0, 1729, 12, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 11, 977, 37, 0, 5, 1, 0, 0, 0, 0, 0, 4, 3254, 0, 2, 0, 11, 125, 4083, 48, 363, 3572, 3845, 0, 0, 0, 0, 0, 574, 103, 159, 0] payload_rows_per_dst=[8112, 7694, 4432, 8170, 2036, 6516, 7420, 5430] dup_rows_per_dst=[5002, 4484, 1474, 80, 26, 4, 4116, 540] combine_rows_per_src=[8112, 7694, 4432, 8170, 2036, 6516, 7420, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52054 payload_rows_remote=44354 dup_rows=13482 dup_rows_remote=13456 dedup_ratio=0.205719 router_tokens_per_expert=[2, 6, 0, 0, 2397, 0, 216, 1, 1910, 0, 0, 540, 3, 1379, 1, 157, 194, 0, 3149, 0, 2, 2, 0, 1, 0, 5, 0, 7, 0, 1, 0, 0, 0, 3368, 13, 0, 0, 111, 9, 0, 2, 2, 1, 0, 3836, 0, 22, 0, 0, 14, 4006, 1353, 1, 1644, 4029, 0, 0, 75, 0, 0, 2334, 1926, 49, 0] payload_rows_per_dst=[4938, 3872, 6404, 8024, 7504, 7700, 5438, 8174] dup_rows_per_dst=[306, 288, 292, 14, 3318, 26, 586, 8652] combine_rows_per_src=[4938, 3872, 6404, 8024, 7504, 7700, 5438, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54296 payload_rows_remote=47362 dup_rows=11240 dup_rows_remote=8534 dedup_ratio=0.171509 router_tokens_per_expert=[0, 0, 0, 0, 3566, 0, 879, 0, 1790, 0, 0, 438, 6, 3279, 0, 279, 826, 2, 2874, 54, 31, 0, 0, 0, 1, 0, 0, 7, 0, 1, 0, 0, 0, 2863, 0, 4, 0, 125, 38, 0, 0, 0, 0, 0, 3483, 2, 0, 0, 8, 0, 4057, 2412, 0, 3309, 939, 0, 0, 1026, 0, 0, 91, 6, 372, 0] payload_rows_per_dst=[7428, 7012, 6040, 8122, 6934, 6966, 7748, 4046] dup_rows_per_dst=[1462, 992, 1534, 10, 2706, 4, 3710, 822] combine_rows_per_src=[7428, 7012, 6040, 8122, 6934, 6966, 7748, 4046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54396 payload_rows_remote=47606 dup_rows=11140 dup_rows_remote=10286 dedup_ratio=0.169983 router_tokens_per_expert=[0, 6, 0, 3, 2486, 0, 3204, 4, 3178, 0, 4, 2594, 0, 639, 3, 173, 568, 4, 3191, 59, 0, 0, 0, 0, 0, 2, 1, 118, 0, 0, 0, 5, 7, 768, 0, 0, 0, 136, 161, 0, 0, 0, 5, 22, 3757, 0, 0, 0, 17, 47, 4096, 2320, 0, 755, 3512, 0, 0, 37, 0, 0, 764, 109, 12, 1] payload_rows_per_dst=[7712, 6314, 6790, 8192, 4804, 7558, 5702, 7324] dup_rows_per_dst=[3694, 3038, 854, 252, 1170, 10, 576, 1546] combine_rows_per_src=[7712, 6314, 6790, 8192, 4804, 7558, 5702, 7324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=182796288 dispatch_meta_bytes=229312 combine_payload_bytes=182796288 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=183025600 ingress_bytes=233127936 link_bytes=233127936 token_bytes=365821888 weight_movement_bytes=50331648 total_nvlink_bytes=416153536 weight_to_token_ratio=0.137585 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.773834229e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53960 payload_rows_remote=49530 dup_rows=11576 dup_rows_remote=11336 dedup_ratio=0.176636 router_tokens_per_expert=[0, 0, 2, 1, 3488, 0, 3263, 0, 460, 0, 0, 450, 0, 1722, 0, 83, 1391, 23, 3434, 0, 0, 0, 0, 1, 0, 0, 0, 0, 60, 6, 0, 1, 0, 3406, 0, 0, 0, 133, 111, 1, 0, 0, 0, 8, 3787, 0, 0, 0, 17, 13, 4040, 1518, 0, 785, 3974, 2, 0, 89, 1, 0, 403, 65, 30, 0] payload_rows_per_dst=[6760, 4050, 7370, 8178, 7206, 7866, 4430, 8100] dup_rows_per_dst=[230, 460, 2328, 36, 1016, 6242, 240, 1024] combine_rows_per_src=[6760, 4050, 7370, 8178, 7206, 7866, 4430, 8100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=183934976 dispatch_meta_bytes=225128 combine_payload_bytes=183934976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=184160104 ingress_bytes=209100800 link_bytes=209100800 token_bytes=368095080 weight_movement_bytes=25165824 total_nvlink_bytes=393260904 weight_to_token_ratio=0.068368 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591015485e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=172507136 dispatch_meta_bytes=213432 combine_payload_bytes=172507136 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=172720568 ingress_bytes=222838784 link_bytes=222838784 token_bytes=345227704 weight_movement_bytes=50331648 total_nvlink_bytes=395559352 weight_to_token_ratio=0.145793 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.695545670e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=181673984 dispatch_meta_bytes=231240 combine_payload_bytes=181673984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=181905224 ingress_bytes=206839808 link_bytes=206839808 token_bytes=363579208 weight_movement_bytes=25165824 total_nvlink_bytes=388745032 weight_to_token_ratio=0.069217 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=27520 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.573811949e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193994752 dispatch_meta_bytes=223584 combine_payload_bytes=193994752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=194218336 ingress_bytes=219160576 link_bytes=219160576 token_bytes=388213088 weight_movement_bytes=25165824 total_nvlink_bytes=413378912 weight_to_token_ratio=0.064825 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.667558757e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 2, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=194994176 dispatch_meta_bytes=231568 combine_payload_bytes=194994176 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=195225744 ingress_bytes=220160000 link_bytes=220160000 token_bytes=390219920 weight_movement_bytes=25165824 total_nvlink_bytes=415385744 weight_to_token_ratio=0.064491 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.675163219e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=202874880 dispatch_meta_bytes=243464 combine_payload_bytes=202874880 prefetch_bytes=25165824 reduce_bytes=125829120 egress_bytes=203118344 ingress_bytes=353869824 link_bytes=353869824 token_bytes=405993224 weight_movement_bytes=150994944 total_nvlink_bytes=556988168 weight_to_token_ratio=0.371915 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.692540487e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46836 payload_rows_remote=43502 dup_rows=18700 dup_rows_remote=18448 dedup_ratio=0.285339 router_tokens_per_expert=[640, 0, 0, 0, 0, 73, 1, 11, 0, 22, 3650, 809, 0, 0, 2109, 0, 0, 0, 229, 1, 555, 25, 24, 0, 0, 3210, 0, 0, 3, 2826, 412, 0, 0, 0, 0, 0, 2093, 4015, 1068, 0, 912, 0, 102, 0, 0, 0, 28, 0, 0, 1, 61, 456, 1191, 0, 0, 0, 914, 0, 2057, 819, 3088, 0, 1363, 0] payload_rows_per_dst=[3334, 8044, 8112, 7828, 4186, 2070, 7034, 6228] dup_rows_per_dst=[252, 5960, 1586, 5052, 0, 14, 2560, 3276] combine_rows_per_src=[3334, 8044, 8112, 7828, 4186, 2070, 7034, 6228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55636 payload_rows_remote=47444 dup_rows=9900 dup_rows_remote=7904 dedup_ratio=0.151062 router_tokens_per_expert=[0, 0, 0, 0, 0, 2453, 0, 1, 0, 1795, 4, 0, 0, 3987, 0, 0, 0, 0, 2653, 3156, 0, 2, 0, 0, 0, 0, 0, 6, 0, 1002, 0, 4086, 0, 0, 2310, 0, 0, 0, 0, 0, 1, 0, 3490, 0, 0, 0, 0, 0, 0, 3304, 1, 0, 4042, 0, 92, 0, 0, 0, 0, 0, 383, 0, 0, 0] payload_rows_per_dst=[7588, 8076, 5310, 8192, 4620, 6982, 6784, 8084] dup_rows_per_dst=[3632, 3496, 0, 1996, 0, 0, 10, 766] combine_rows_per_src=[7588, 8076, 5310, 8192, 4620, 6982, 6784, 8084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51240 payload_rows_remote=45704 dup_rows=14296 dup_rows_remote=13064 dedup_ratio=0.218140 router_tokens_per_expert=[5, 0, 0, 0, 0, 2115, 0, 2, 0, 52, 16, 0, 0, 3688, 0, 32, 0, 0, 3822, 3757, 0, 29, 0, 0, 0, 0, 0, 0, 7, 1315, 0, 4095, 0, 0, 3355, 0, 0, 0, 0, 0, 0, 0, 4079, 42, 0, 0, 1, 0, 0, 2942, 0, 0, 2421, 0, 30, 0, 0, 0, 0, 0, 951, 0, 0, 12] payload_rows_per_dst=[4240, 7398, 7572, 2644, 8190, 8178, 7482, 5536] dup_rows_per_dst=[4, 178, 0, 0, 6710, 2222, 3950, 1232] combine_rows_per_src=[4240, 7398, 7572, 2644, 8190, 8178, 7482, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51516 payload_rows_remote=43370 dup_rows=14020 dup_rows_remote=9710 dedup_ratio=0.213928 router_tokens_per_expert=[0, 0, 0, 0, 0, 2114, 1, 1, 0, 1995, 156, 0, 0, 4048, 0, 29, 0, 19, 655, 3813, 0, 0, 0, 0, 0, 0, 0, 2, 14, 1932, 0, 3272, 0, 10, 3451, 0, 0, 0, 0, 0, 314, 0, 1564, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 3885, 0, 21, 0, 0, 13, 0, 2, 1365, 0, 0, 0] payload_rows_per_dst=[8036, 8146, 1348, 7030, 6904, 3756, 8182, 8114] dup_rows_per_dst=[3822, 4310, 0, 3410, 18, 2, 42, 2416] combine_rows_per_src=[8036, 8146, 1348, 7030, 6904, 3756, 8182, 8114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55744 payload_rows_remote=49678 dup_rows=9792 dup_rows_remote=9792 dedup_ratio=0.149414 router_tokens_per_expert=[0, 0, 0, 0, 0, 2219, 0, 0, 0, 708, 1, 0, 0, 4030, 0, 1, 0, 0, 3181, 3596, 5, 0, 0, 0, 0, 0, 13, 15, 1, 1244, 0, 4094, 1, 0, 3019, 0, 13, 0, 0, 0, 0, 0, 4014, 4, 0, 0, 0, 0, 0, 2301, 0, 0, 4062, 0, 86, 0, 0, 0, 0, 0, 158, 2, 0, 0] payload_rows_per_dst=[4474, 8082, 7998, 8192, 6066, 8036, 4724, 8172] dup_rows_per_dst=[294, 1398, 5236, 2542, 0, 0, 50, 272] combine_rows_per_src=[4474, 8082, 7998, 8192, 6066, 8036, 4724, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=178184192 dispatch_meta_bytes=247800 combine_payload_bytes=178184192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=178431992 ingress_bytes=203350016 link_bytes=203350016 token_bytes=356616184 weight_movement_bytes=25165824 total_nvlink_bytes=381782008 weight_to_token_ratio=0.070568 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.547258664e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=194330624 dispatch_meta_bytes=221392 combine_payload_bytes=194330624 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=194552016 ingress_bytes=269828096 link_bytes=269828096 token_bytes=388882640 weight_movement_bytes=75497472 total_nvlink_bytes=464380112 weight_to_token_ratio=0.194139 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.053080041e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54814 payload_rows_remote=46632 dup_rows=10722 dup_rows_remote=10642 dedup_ratio=0.163605 router_tokens_per_expert=[2, 1, 4, 0, 0, 2961, 0, 7, 10, 294, 0, 0, 0, 1372, 0, 30, 0, 0, 2975, 3870, 0, 31, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 4088, 0, 0, 3286, 0, 1, 0, 7, 0, 40, 0, 4091, 0, 0, 0, 0, 0, 14, 4084, 0, 0, 4096, 0, 315, 0, 5, 4, 0, 0, 85, 0, 3, 0] payload_rows_per_dst=[5942, 3326, 7876, 5246, 7880, 8182, 8170, 8192] dup_rows_per_dst=[8, 86, 5876, 598, 3224, 80, 656, 194] combine_rows_per_src=[5942, 3326, 7876, 5246, 7880, 8182, 8170, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56978 payload_rows_remote=48880 dup_rows=8558 dup_rows_remote=8516 dedup_ratio=0.130585 router_tokens_per_expert=[1, 0, 4, 0, 0, 1078, 4, 0, 0, 2117, 0, 0, 6, 836, 0, 6, 19, 1, 4044, 4093, 3, 0, 1, 2, 0, 12, 8, 4, 0, 2856, 0, 4091, 0, 0, 1830, 0, 0, 14, 0, 0, 25, 0, 3615, 0, 12, 0, 0, 1, 0, 3951, 14, 0, 4019, 5, 77, 0, 0, 3, 2, 0, 7, 0, 7, 0] payload_rows_per_dst=[8192, 5530, 8098, 8188, 3688, 7306, 7914, 8062] dup_rows_per_dst=[2168, 400, 42, 5754, 0, 0, 180, 14] combine_rows_per_src=[8192, 5530, 8098, 8188, 3688, 7306, 7914, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49458 payload_rows_remote=42534 dup_rows=16078 dup_rows_remote=16036 dedup_ratio=0.245331 router_tokens_per_expert=[12, 0, 0, 0, 0, 516, 1, 0, 0, 29, 62, 0, 0, 4083, 0, 4, 0, 0, 4063, 3943, 0, 1, 0, 0, 0, 0, 1, 0, 1, 229, 0, 4091, 0, 0, 3821, 0, 0, 0, 0, 0, 2, 0, 4050, 0, 0, 0, 0, 0, 0, 3261, 3, 0, 4001, 0, 218, 1, 0, 0, 0, 2, 373, 0, 0, 0] payload_rows_per_dst=[1054, 8172, 8164, 808, 8182, 8104, 6924, 8050] dup_rows_per_dst=[4, 184, 2022, 12, 7642, 5470, 42, 702] combine_rows_per_src=[1054, 8172, 8164, 808, 8182, 8104, 6924, 8050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=187203584 dispatch_meta_bytes=235072 combine_payload_bytes=187203584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=187438656 ingress_bytes=212369408 link_bytes=212369408 token_bytes=374642240 weight_movement_bytes=25165824 total_nvlink_bytes=399808064 weight_to_token_ratio=0.067173 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.615885816e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=177643520 dispatch_meta_bytes=212320 combine_payload_bytes=177643520 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=177855840 ingress_bytes=202809344 link_bytes=202809344 token_bytes=355499360 weight_movement_bytes=25165824 total_nvlink_bytes=380665184 weight_to_token_ratio=0.070790 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.543144775e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=203481088 dispatch_meta_bytes=237880 combine_payload_bytes=203481088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203718968 ingress_bytes=228646912 link_bytes=228646912 token_bytes=407200056 weight_movement_bytes=25165824 total_nvlink_bytes=432365880 weight_to_token_ratio=0.061802 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=22400 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.739738813e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=191004672 dispatch_meta_bytes=229096 combine_payload_bytes=191004672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=191233768 ingress_bytes=216170496 link_bytes=216170496 token_bytes=382238440 weight_movement_bytes=25165824 total_nvlink_bytes=407404264 weight_to_token_ratio=0.065838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=28544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.644807703e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=200212480 dispatch_meta_bytes=229584 combine_payload_bytes=200212480 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=200442064 ingress_bytes=300875776 link_bytes=300875776 token_bytes=400654544 weight_movement_bytes=100663296 total_nvlink_bytes=501317840 weight_to_token_ratio=0.251247 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.289317013e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=174219264 dispatch_meta_bytes=234280 combine_payload_bytes=174219264 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=174453544 ingress_bytes=224550912 link_bytes=224550912 token_bytes=348672808 weight_movement_bytes=50331648 total_nvlink_bytes=399004456 weight_to_token_ratio=0.144352 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.708572986e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54620 payload_rows_remote=48618 dup_rows=10916 dup_rows_remote=9200 dedup_ratio=0.166565 router_tokens_per_expert=[0, 427, 0, 1, 1052, 0, 0, 2354, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 1, 46, 0, 0, 1428, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2399, 3984, 4087, 0, 3612, 889, 0, 1, 0, 0, 15, 2, 15, 1567, 0, 2384, 0, 0, 0, 0, 0, 0, 4059] payload_rows_per_dst=[6396, 8086, 7968, 6002, 8178, 6436, 6350, 5204] dup_rows_per_dst=[1272, 0, 800, 1716, 0, 3176, 2576, 1376] combine_rows_per_src=[6396, 8086, 7968, 6002, 8178, 6436, 6350, 5204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48902 payload_rows_remote=44628 dup_rows=16634 dup_rows_remote=16156 dedup_ratio=0.253815 router_tokens_per_expert=[0, 1, 0, 0, 697, 0, 1678, 0, 3526, 0, 0, 19, 0, 3112, 0, 899, 2967, 5, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 0, 0, 0, 0, 1272, 0, 0, 0, 3, 16, 0, 0, 0, 0, 1, 4039, 0, 1, 0, 6, 0, 4050, 1981, 0, 1022, 3989, 0, 0, 1, 0, 0, 1094, 167, 34, 0] payload_rows_per_dst=[4274, 8092, 7368, 8106, 2578, 8082, 8052, 2350] dup_rows_per_dst=[478, 8744, 5140, 44, 4, 0, 1982, 242] combine_rows_per_src=[4274, 8092, 7368, 8106, 2578, 8082, 8052, 2350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=199139328 dispatch_meta_bytes=231272 combine_payload_bytes=199139328 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199370600 ingress_bytes=224305152 link_bytes=224305152 token_bytes=398509928 weight_movement_bytes=25165824 total_nvlink_bytes=423675752 weight_to_token_ratio=0.063150 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=20608 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.706703036e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52392 payload_rows_remote=44250 dup_rows=13144 dup_rows_remote=12864 dedup_ratio=0.200562 router_tokens_per_expert=[0, 120, 0, 15, 2281, 0, 0, 938, 4087, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 2938, 0, 1, 0, 0, 0, 0, 0, 0, 1, 721, 3759, 4094, 0, 3774, 2048, 0, 0, 1, 0, 0, 949, 0, 2800, 0, 140, 0, 0, 0, 0, 0, 0, 4071] payload_rows_per_dst=[5650, 8174, 54, 5876, 8190, 8128, 8178, 8142] dup_rows_per_dst=[1058, 0, 6, 0, 0, 4930, 6870, 280] combine_rows_per_src=[5650, 8174, 54, 5876, 8190, 8128, 8178, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54612 payload_rows_remote=46422 dup_rows=10924 dup_rows_remote=10892 dedup_ratio=0.166687 router_tokens_per_expert=[38, 2588, 0, 1, 0, 0, 0, 1626, 4087, 0, 23, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 12, 0, 1774, 2, 0, 0, 0, 312, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 6, 0, 0, 34, 4067, 3962, 1, 1870, 3092, 0, 0, 0, 0, 0, 911, 16, 1385, 0, 1704, 0, 0, 0, 0, 0, 0, 1193] payload_rows_per_dst=[7078, 8190, 8190, 8172, 7934, 7790, 4392, 2866] dup_rows_per_dst=[1970, 32, 3528, 2956, 2, 2204, 232, 0] combine_rows_per_src=[7078, 8190, 8190, 8172, 7934, 7790, 4392, 2866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52922 payload_rows_remote=44884 dup_rows=12614 dup_rows_remote=12614 dedup_ratio=0.192474 router_tokens_per_expert=[5, 0, 0, 0, 771, 0, 0, 1607, 4054, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 0, 0, 0, 3, 0, 0, 1773, 0, 0, 10, 0, 0, 0, 0, 1, 11, 24, 4038, 4008, 0, 4054, 2260, 0, 0, 0, 0, 0, 211, 28, 1746, 0, 2977, 0, 0, 0, 0, 0, 0, 4038] payload_rows_per_dst=[4262, 8126, 8080, 3546, 8038, 4542, 8174, 8154] dup_rows_per_dst=[504, 0, 2276, 6, 0, 48, 3904, 5876] combine_rows_per_src=[4262, 8126, 8080, 3546, 8038, 4542, 8174, 8154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53550 payload_rows_remote=52544 dup_rows=11986 dup_rows_remote=11846 dedup_ratio=0.182892 router_tokens_per_expert=[3, 14, 0, 0, 2653, 0, 0, 201, 4068, 10, 0, 8, 0, 3, 3, 0, 2, 0, 0, 9, 7, 0, 0, 1078, 0, 6, 0, 0, 0, 11, 0, 3257, 0, 1, 0, 12, 0, 0, 0, 0, 0, 312, 4088, 4081, 0, 4075, 261, 0, 2, 0, 0, 2, 996, 0, 1846, 0, 1956, 2, 0, 0, 7, 0, 0, 3794] payload_rows_per_dst=[5624, 8170, 8192, 6540, 8188, 1006, 8182, 7648] dup_rows_per_dst=[118, 14, 2176, 8, 0, 140, 5660, 3870] combine_rows_per_src=[5624, 8170, 8192, 6540, 8188, 1006, 8182, 7648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=182796288 dispatch_meta_bytes=243136 combine_payload_bytes=182796288 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=183039424 ingress_bytes=233127936 link_bytes=233127936 token_bytes=365835712 weight_movement_bytes=50331648 total_nvlink_bytes=416167360 weight_to_token_ratio=0.137580 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.773834229e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55806 payload_rows_remote=47648 dup_rows=9730 dup_rows_remote=8342 dedup_ratio=0.148468 router_tokens_per_expert=[5, 711, 0, 0, 1626, 0, 0, 1804, 4093, 7, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 693, 0, 0, 0, 48, 104, 10, 2, 2872, 1, 0, 3, 0, 0, 0, 0, 3, 0, 387, 4073, 3976, 15, 4072, 12, 0, 0, 0, 1, 0, 19, 3, 509, 0, 3621, 0, 0, 0, 0, 0, 0, 4091] payload_rows_per_dst=[6842, 8190, 8158, 8192, 7962, 8168, 1052, 7242] dup_rows_per_dst=[1450, 10, 1388, 6062, 4, 804, 12, 0] combine_rows_per_src=[6842, 8190, 8158, 8192, 7962, 8168, 1052, 7242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 2, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53836 payload_rows_remote=46092 dup_rows=11700 dup_rows_remote=10174 dedup_ratio=0.178528 router_tokens_per_expert=[0, 289, 0, 1, 2887, 0, 0, 748, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 243, 0, 1, 0, 0, 15, 3, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1672, 4094, 4093, 0, 3738, 1310, 0, 3, 0, 0, 0, 111, 0, 783, 0, 3585, 1, 2, 0, 0, 0, 0, 3851] payload_rows_per_dst=[6740, 8188, 5896, 2500, 8186, 6486, 7744, 8096] dup_rows_per_dst=[1110, 0, 448, 20, 0, 1814, 1526, 6782] combine_rows_per_src=[6740, 8188, 5896, 2500, 8186, 6486, 7744, 8096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=181248000 dispatch_meta_bytes=228456 combine_payload_bytes=181248000 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=181476456 ingress_bytes=281911296 link_bytes=281911296 token_bytes=362724456 weight_movement_bytes=100663296 total_nvlink_bytes=463387752 weight_to_token_ratio=0.277520 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.145019232e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=190144512 dispatch_meta_bytes=229256 combine_payload_bytes=190144512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=190373768 ingress_bytes=215310336 link_bytes=215310336 token_bytes=380518280 weight_movement_bytes=25165824 total_nvlink_bytes=405684104 weight_to_token_ratio=0.066136 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.638262880e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=215220224 dispatch_meta_bytes=257560 combine_payload_bytes=215220224 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=215477784 ingress_bytes=341049344 link_bytes=341049344 token_bytes=430698008 weight_movement_bytes=125829120 total_nvlink_bytes=556527128 weight_to_token_ratio=0.292152 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=10880 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.594991447e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=183844864 dispatch_meta_bytes=229992 combine_payload_bytes=183844864 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=184074856 ingress_bytes=234176512 link_bytes=234176512 token_bytes=367919720 weight_movement_bytes=50331648 total_nvlink_bytes=418251368 weight_to_token_ratio=0.136801 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781812680e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=195166208 dispatch_meta_bytes=223960 combine_payload_bytes=195166208 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=195390168 ingress_bytes=220332032 link_bytes=220332032 token_bytes=390556376 weight_movement_bytes=25165824 total_nvlink_bytes=415722200 weight_to_token_ratio=0.064436 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.676472184e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=188792832 dispatch_meta_bytes=225064 combine_payload_bytes=188792832 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189017896 ingress_bytes=213958656 link_bytes=213958656 token_bytes=377810728 weight_movement_bytes=25165824 total_nvlink_bytes=402976552 weight_to_token_ratio=0.066610 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.627978157e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49998 payload_rows_remote=42376 dup_rows=15538 dup_rows_remote=10978 dedup_ratio=0.237091 router_tokens_per_expert=[0, 0, 0, 3685, 0, 29, 0, 0, 0, 0, 0, 4061, 0, 166, 0, 0, 0, 43, 3053, 51, 0, 0, 0, 1, 225, 0, 157, 0, 0, 2849, 0, 0, 980, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 1, 3114, 737, 0, 4056, 0, 0, 3888, 0, 0, 0, 0, 0, 2567, 0, 0, 2969, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8122, 7514, 7622, 2180, 3430, 7776, 5168] dup_rows_per_dst=[7354, 332, 3020, 4560, 52, 36, 0, 184] combine_rows_per_src=[8186, 8122, 7514, 7622, 2180, 3430, 7776, 5168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50608 payload_rows_remote=42418 dup_rows=14928 dup_rows_remote=7040 dedup_ratio=0.227783 router_tokens_per_expert=[0, 0, 0, 0, 0, 3972, 0, 0, 0, 1936, 0, 0, 0, 1050, 0, 0, 0, 0, 2996, 4067, 0, 1, 0, 0, 0, 0, 0, 2, 0, 1760, 0, 4091, 0, 0, 1204, 0, 0, 0, 0, 2, 0, 0, 3607, 2, 0, 0, 0, 0, 0, 3945, 0, 0, 3762, 0, 303, 0, 0, 0, 0, 0, 68, 0, 0, 0] payload_rows_per_dst=[8190, 5422, 5994, 7628, 2412, 7218, 8018, 5726] dup_rows_per_dst=[7888, 1332, 0, 3296, 0, 0, 2408, 4] combine_rows_per_src=[8190, 5422, 5994, 7628, 2412, 7218, 8018, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=173572096 dispatch_meta_bytes=213416 combine_payload_bytes=173572096 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=173785512 ingress_bytes=198737920 link_bytes=198737920 token_bytes=347357608 weight_movement_bytes=25165824 total_nvlink_bytes=372523432 weight_to_token_ratio=0.072449 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.512165942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=42038 payload_rows_remote=33846 dup_rows=23498 dup_rows_remote=15642 dedup_ratio=0.358551 router_tokens_per_expert=[0, 0, 0, 53, 0, 53, 0, 0, 3, 0, 0, 4073, 0, 436, 10, 0, 1, 59, 3877, 0, 0, 0, 0, 0, 201, 78, 0, 1, 0, 473, 0, 0, 2251, 0, 0, 0, 0, 13, 1485, 5, 0, 0, 0, 0, 3026, 1114, 0, 3575, 0, 0, 4086, 1, 0, 0, 0, 0, 4096, 4, 1, 3793, 0, 0, 0, 0] payload_rows_per_dst=[212, 22, 7756, 1350, 8152, 8164, 8190, 8192] dup_rows_per_dst=[0, 4, 118, 156, 7242, 7266, 856, 7856] combine_rows_per_src=[212, 22, 7756, 1350, 8152, 8164, 8190, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=42096 payload_rows_remote=33922 dup_rows=23440 dup_rows_remote=17210 dedup_ratio=0.357666 router_tokens_per_expert=[0, 1, 0, 3902, 1, 13, 4, 0, 0, 0, 0, 4077, 0, 3125, 0, 0, 0, 3, 97, 180, 0, 0, 0, 13, 3907, 3, 222, 0, 0, 872, 0, 0, 2760, 0, 0, 0, 0, 16, 2, 0, 1, 0, 0, 2, 1627, 52, 0, 3652, 0, 0, 3976, 0, 0, 0, 0, 0, 166, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8166, 8174, 3624, 8192, 5546, 110, 7952, 332] dup_rows_per_dst=[6980, 6230, 216, 10004, 10, 0, 0, 0] combine_rows_per_src=[8166, 8174, 3624, 8192, 5546, 110, 7952, 332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52414 payload_rows_remote=44266 dup_rows=13122 dup_rows_remote=8402 dedup_ratio=0.200226 router_tokens_per_expert=[4, 0, 0, 79, 0, 0, 0, 0, 0, 13, 0, 4086, 0, 3573, 0, 0, 7, 223, 3638, 0, 4, 0, 249, 0, 90, 0, 0, 0, 0, 1525, 4, 8, 1628, 9, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 3900, 41, 8, 3281, 0, 1, 4089, 0, 3, 0, 1, 8, 2817, 0, 0, 3405, 0, 7, 0, 0] payload_rows_per_dst=[1740, 7152, 7862, 3234, 8190, 8148, 8192, 7896] dup_rows_per_dst=[18, 20, 380, 20, 3390, 4720, 12, 4562] combine_rows_per_src=[1740, 7152, 7862, 3234, 8190, 8148, 8192, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50532 payload_rows_remote=43504 dup_rows=15004 dup_rows_remote=11200 dedup_ratio=0.228943 router_tokens_per_expert=[1, 0, 0, 3067, 0, 64, 0, 0, 0, 0, 0, 4094, 0, 2665, 1, 0, 0, 734, 466, 25, 0, 0, 0, 0, 202, 58, 8, 0, 0, 1139, 0, 0, 1001, 0, 0, 0, 0, 17, 1237, 0, 0, 2, 0, 1, 4004, 116, 0, 3898, 0, 0, 3853, 0, 0, 0, 0, 0, 4063, 0, 0, 2052, 0, 0, 0, 0] payload_rows_per_dst=[8162, 6658, 2382, 2426, 7028, 8020, 7706, 8150] dup_rows_per_dst=[5898, 540, 68, 388, 3804, 226, 0, 4080] combine_rows_per_src=[8162, 6658, 2382, 2426, 7028, 8020, 7706, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49726 payload_rows_remote=41828 dup_rows=15810 dup_rows_remote=11104 dedup_ratio=0.241241 router_tokens_per_expert=[0, 4, 12, 1242, 0, 0, 8, 22, 0, 0, 0, 4045, 0, 3740, 0, 0, 0, 0, 2786, 0, 0, 0, 92, 0, 1199, 894, 165, 0, 0, 173, 0, 0, 2291, 0, 0, 0, 11, 0, 160, 0, 0, 0, 4, 0, 3424, 579, 0, 3136, 0, 0, 3460, 0, 0, 1, 1, 0, 3744, 0, 0, 1575, 0, 0, 0, 0] payload_rows_per_dst=[7404, 8090, 7898, 5952, 4806, 1166, 6922, 7488] dup_rows_per_dst=[1444, 7480, 4706, 2060, 118, 0, 2, 0] combine_rows_per_src=[7404, 8090, 7898, 5952, 4806, 1166, 6922, 7488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=173744128 dispatch_meta_bytes=197832 combine_payload_bytes=173744128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=173941960 ingress_bytes=198909952 link_bytes=198909952 token_bytes=347686088 weight_movement_bytes=25165824 total_nvlink_bytes=372851912 weight_to_token_ratio=0.072381 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=17536 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.513474907e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43832 payload_rows_remote=37158 dup_rows=21704 dup_rows_remote=18008 dedup_ratio=0.331177 router_tokens_per_expert=[0, 0, 0, 1123, 0, 40, 0, 0, 0, 0, 0, 4035, 2, 2585, 2, 1, 0, 209, 2030, 34, 0, 1, 6, 0, 1439, 10, 2, 0, 0, 1947, 0, 0, 2864, 0, 0, 0, 0, 6, 369, 0, 0, 0, 0, 0, 3347, 56, 0, 4092, 1, 1, 3287, 0, 0, 0, 0, 1, 2072, 0, 0, 3205, 0, 1, 0, 0] payload_rows_per_dst=[2326, 1386, 4318, 5486, 8098, 8192, 6674, 7352] dup_rows_per_dst=[0, 4, 242, 1310, 6450, 6798, 3696, 3204] combine_rows_per_src=[2326, 1386, 4318, 5486, 8098, 8192, 6674, 7352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=138633216 dispatch_meta_bytes=197952 combine_payload_bytes=138633216 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=138831168 ingress_bytes=214130688 link_bytes=214130688 token_bytes=277464384 weight_movement_bytes=75497472 total_nvlink_bytes=352961856 weight_to_token_ratio=0.272098 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=20096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.629287121e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=138944512 dispatch_meta_bytes=204528 combine_payload_bytes=138944512 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=139149040 ingress_bytes=214441984 link_bytes=214441984 token_bytes=278093552 weight_movement_bytes=75497472 total_nvlink_bytes=353591024 weight_to_token_ratio=0.271482 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=17280 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.631655724e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=181313536 dispatch_meta_bytes=210672 combine_payload_bytes=181313536 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=181524208 ingress_bytes=256811008 link_bytes=256811008 token_bytes=362837744 weight_movement_bytes=75497472 total_nvlink_bytes=438335216 weight_to_token_ratio=0.208075 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.954035042e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=178192384 dispatch_meta_bytes=218816 combine_payload_bytes=178192384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=178411200 ingress_bytes=203358208 link_bytes=203358208 token_bytes=356603584 weight_movement_bytes=25165824 total_nvlink_bytes=381769408 weight_to_token_ratio=0.070571 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=15232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.547320995e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=171327488 dispatch_meta_bytes=211728 combine_payload_bytes=171327488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=171539216 ingress_bytes=196493312 link_bytes=196493312 token_bytes=342866704 weight_movement_bytes=25165824 total_nvlink_bytes=368032528 weight_to_token_ratio=0.073398 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.495087069e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60330 payload_rows_remote=53460 dup_rows=5206 dup_rows_remote=5206 dedup_ratio=0.079437 router_tokens_per_expert=[0, 0, 0, 0, 2853, 6, 0, 0, 0, 0, 0, 0, 881, 1564, 0, 0, 5, 0, 0, 0, 0, 0, 0, 179, 0, 1, 4095, 4096, 0, 3434, 0, 3422, 0, 0, 0, 0, 4096, 0, 3442, 0, 0, 7, 10, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 577, 4090, 0, 0, 0, 0, 0, 0, 8, 0, 0] payload_rows_per_dst=[5718, 8090, 8192, 6870, 6884, 8192, 8192, 8192] dup_rows_per_dst=[0, 3644, 368, 0, 0, 36, 1142, 16] combine_rows_per_src=[5718, 8090, 8192, 6870, 6884, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=152199168 dispatch_meta_bytes=220664 combine_payload_bytes=152199168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=152419832 ingress_bytes=177364992 link_bytes=177364992 token_bytes=304619000 weight_movement_bytes=25165824 total_nvlink_bytes=329784824 weight_to_token_ratio=0.082614 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.349542655e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49418 payload_rows_remote=41252 dup_rows=16118 dup_rows_remote=8822 dedup_ratio=0.245941 router_tokens_per_expert=[0, 384, 0, 0, 3243, 0, 0, 763, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4077, 4096, 0, 3937, 1268, 0, 0, 0, 0, 0, 266, 0, 76, 0, 3341, 0, 0, 0, 0, 0, 0, 4049] payload_rows_per_dst=[8166, 8192, 8168, 8014, 8192, 8002, 684, 0] dup_rows_per_dst=[7296, 164, 208, 5926, 662, 1862, 0, 0] combine_rows_per_src=[8166, 8192, 8168, 8014, 8192, 8002, 684, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=218972160 dispatch_meta_bytes=234664 combine_payload_bytes=218972160 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=219206824 ingress_bytes=344801280 link_bytes=344801280 token_bytes=438178984 weight_movement_bytes=125829120 total_nvlink_bytes=564008104 weight_to_token_ratio=0.287164 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22144 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.623539345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54816 payload_rows_remote=46624 dup_rows=10720 dup_rows_remote=9400 dedup_ratio=0.163574 router_tokens_per_expert=[0, 0, 4, 0, 2965, 177, 0, 0, 0, 0, 0, 0, 343, 681, 0, 0, 0, 0, 0, 7, 0, 0, 0, 376, 0, 0, 3935, 4093, 2, 1090, 0, 3931, 1, 0, 0, 0, 4096, 0, 3750, 5, 2, 1, 255, 0, 0, 0, 13, 9, 0, 0, 0, 0, 0, 2289, 4083, 0, 0, 0, 0, 0, 0, 660, 0, 0] payload_rows_per_dst=[6290, 1578, 7004, 7898, 7512, 8160, 8182, 8192] dup_rows_per_dst=[2, 470, 760, 2694, 0, 270, 5204, 1320] combine_rows_per_src=[6290, 1578, 7004, 7898, 7512, 8160, 8182, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60866 payload_rows_remote=53104 dup_rows=4670 dup_rows_remote=4670 dedup_ratio=0.071259 router_tokens_per_expert=[2, 0, 5, 0, 2690, 5, 0, 0, 0, 0, 0, 0, 2, 738, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 3590, 2, 3828, 2, 0, 0, 0, 4095, 0, 3879, 0, 0, 0, 1, 0, 0, 0, 35, 9, 0, 0, 24, 0, 0, 297, 4093, 0, 0, 0, 0, 0, 0, 1273, 0, 0] payload_rows_per_dst=[5404, 7754, 8192, 7184, 7762, 8192, 8188, 8190] dup_rows_per_dst=[0, 1382, 28, 0, 0, 74, 640, 2546] combine_rows_per_src=[5404, 7754, 8192, 7184, 7762, 8192, 8188, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54118 payload_rows_remote=45938 dup_rows=11418 dup_rows_remote=3276 dedup_ratio=0.174225 router_tokens_per_expert=[7, 0, 0, 0, 1767, 674, 0, 0, 0, 0, 1, 0, 1036, 3393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4095, 4082, 0, 3411, 0, 3731, 0, 0, 0, 0, 4071, 0, 2724, 2, 0, 5, 4, 0, 0, 0, 54, 23, 0, 0, 0, 0, 0, 5, 3629, 0, 39, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8096, 8180, 8188, 6822, 5448, 8192, 1010, 8182] dup_rows_per_dst=[3054, 8142, 6, 0, 4, 170, 4, 38] combine_rows_per_src=[8096, 8180, 8188, 6822, 5448, 8192, 1010, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59892 payload_rows_remote=51714 dup_rows=5644 dup_rows_remote=5536 dedup_ratio=0.086121 router_tokens_per_expert=[18, 0, 0, 16, 3639, 292, 0, 0, 12, 0, 3, 0, 0, 1262, 0, 19, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4081, 0, 2518, 0, 3335, 11, 0, 10, 0, 4083, 3, 4061, 0, 0, 0, 10, 0, 43, 0, 9, 0, 0, 0, 0, 0, 13, 1008, 4087, 6, 0, 0, 7, 4, 0, 113, 8, 14] payload_rows_per_dst=[7794, 5826, 8166, 5386, 8158, 8178, 8192, 8192] dup_rows_per_dst=[136, 1458, 0, 1628, 12, 108, 2036, 266] combine_rows_per_src=[7794, 5826, 8166, 5386, 8158, 8178, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61096 payload_rows_remote=52906 dup_rows=4440 dup_rows_remote=4410 dedup_ratio=0.067749 router_tokens_per_expert=[88, 2, 0, 0, 3348, 798, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 17, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3998, 4087, 0, 3567, 0, 2600, 0, 0, 0, 0, 4096, 0, 4048, 0, 0, 0, 49, 0, 0, 0, 0, 9, 1, 0, 7, 0, 0, 1056, 4034, 0, 0, 0, 0, 0, 0, 620, 0, 0] payload_rows_per_dst=[7688, 5616, 8190, 7134, 8096, 8002, 8178, 8192] dup_rows_per_dst=[784, 258, 30, 0, 0, 110, 2018, 1240] combine_rows_per_src=[7688, 5616, 8190, 7134, 8096, 8002, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=168968192 dispatch_meta_bytes=200296 combine_payload_bytes=168968192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=169168488 ingress_bytes=194134016 link_bytes=194134016 token_bytes=338136680 weight_movement_bytes=25165824 total_nvlink_bytes=363302504 weight_to_token_ratio=0.074425 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=14592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.477135552e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=190971904 dispatch_meta_bytes=224096 combine_payload_bytes=190971904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=191196000 ingress_bytes=216137728 link_bytes=216137728 token_bytes=382167904 weight_movement_bytes=25165824 total_nvlink_bytes=407333728 weight_to_token_ratio=0.065850 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.644558377e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58290 payload_rows_remote=50098 dup_rows=7246 dup_rows_remote=5836 dedup_ratio=0.110565 router_tokens_per_expert=[2, 0, 0, 0, 3296, 471, 0, 1, 0, 0, 0, 0, 286, 2112, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 0, 0, 4096, 4093, 2, 719, 0, 2563, 0, 0, 0, 0, 4095, 1, 4033, 0, 0, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 705, 4095, 0, 0, 0, 0, 0, 0, 8, 0, 1] payload_rows_per_dst=[7324, 4262, 8192, 5868, 8068, 8192, 8192, 8192] dup_rows_per_dst=[216, 534, 18, 700, 0, 4352, 1410, 16] combine_rows_per_src=[7324, 4262, 8192, 5868, 8068, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=217513984 dispatch_meta_bytes=231096 combine_payload_bytes=217513984 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=217745080 ingress_bytes=267845632 link_bytes=267845632 token_bytes=435259064 weight_movement_bytes=50331648 total_nvlink_bytes=485590712 weight_to_token_ratio=0.115636 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=29952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.037995781e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=188162048 dispatch_meta_bytes=196856 combine_payload_bytes=188162048 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188358904 ingress_bytes=213327872 link_bytes=213327872 token_bytes=376520952 weight_movement_bytes=25165824 total_nvlink_bytes=401686776 weight_to_token_ratio=0.066838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=20096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.623178619e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=216702976 dispatch_meta_bytes=229264 combine_payload_bytes=216702976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=216932240 ingress_bytes=241868800 link_bytes=241868800 token_bytes=433635216 weight_movement_bytes=25165824 total_nvlink_bytes=458801040 weight_to_token_ratio=0.058035 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.840342104e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60374 payload_rows_remote=52182 dup_rows=5162 dup_rows_remote=5118 dedup_ratio=0.078766 router_tokens_per_expert=[0, 0, 0, 121, 0, 0, 556, 3, 0, 0, 0, 0, 0, 0, 1993, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 17, 5, 0, 0, 0, 0, 0, 0, 4095, 4096, 3656, 2, 0, 0, 0, 26, 0, 0, 0, 3995, 1525, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4055, 410] payload_rows_per_dst=[8192, 3986, 8190, 8192, 7316, 8176, 8192, 8130] dup_rows_per_dst=[1358, 0, 2, 44, 0, 2916, 42, 800] combine_rows_per_src=[8192, 3986, 8190, 8192, 7316, 8176, 8192, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=211820544 dispatch_meta_bytes=229000 combine_payload_bytes=211820544 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=212049544 ingress_bytes=236986368 link_bytes=236986368 token_bytes=423870088 weight_movement_bytes=25165824 total_nvlink_bytes=449035912 weight_to_token_ratio=0.059372 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=30080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.803192438e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=205201408 dispatch_meta_bytes=223736 combine_payload_bytes=205201408 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=205425144 ingress_bytes=255533056 link_bytes=255533056 token_bytes=410626552 weight_movement_bytes=50331648 total_nvlink_bytes=460958200 weight_to_token_ratio=0.122573 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.944311304e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48044 payload_rows_remote=40526 dup_rows=17492 dup_rows_remote=17364 dedup_ratio=0.266907 router_tokens_per_expert=[0, 0, 0, 78, 0, 4, 0, 0, 0, 0, 0, 4092, 0, 1569, 0, 0, 0, 1, 3903, 10, 0, 0, 0, 0, 14, 442, 2, 0, 0, 2923, 0, 0, 229, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3802, 1131, 0, 3741, 0, 0, 3779, 0, 0, 0, 0, 0, 4035, 0, 0, 3007, 0, 0, 0, 0] payload_rows_per_dst=[7518, 8188, 8178, 8018, 468, 7044, 7558, 1072] dup_rows_per_dst=[128, 3134, 7254, 4758, 2, 2216, 0, 0] combine_rows_per_src=[7518, 8188, 8178, 8018, 468, 7044, 7558, 1072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=213737472 dispatch_meta_bytes=229200 combine_payload_bytes=213737472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213966672 ingress_bytes=238903296 link_bytes=238903296 token_bytes=427704144 weight_movement_bytes=25165824 total_nvlink_bytes=452869968 weight_to_token_ratio=0.058839 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.817778045e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57652 payload_rows_remote=49490 dup_rows=7884 dup_rows_remote=5932 dedup_ratio=0.120300 router_tokens_per_expert=[0, 0, 0, 52, 0, 0, 91, 11, 2, 0, 2, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 13, 0, 16, 4095, 0, 0, 0, 18, 27, 0, 0, 0, 1, 0, 0, 4079, 4048, 4023, 6, 0, 0, 0, 102, 0, 0, 0, 3292, 29, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 4051, 721] payload_rows_per_dst=[2746, 7978, 7858, 7804, 8180, 6816, 8108, 8162] dup_rows_per_dst=[42, 4, 54, 90, 4720, 184, 838, 1952] combine_rows_per_src=[2746, 7978, 7858, 7804, 8180, 6816, 8108, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61732 payload_rows_remote=54446 dup_rows=3804 dup_rows_remote=3804 dedup_ratio=0.058044 router_tokens_per_expert=[0, 0, 0, 29, 0, 0, 2, 32, 0, 0, 1, 0, 0, 0, 3415, 0, 0, 0, 0, 0, 2, 0, 0, 4071, 0, 0, 0, 18, 50, 0, 0, 0, 0, 0, 0, 4055, 4070, 3642, 1, 0, 0, 1, 148, 0, 0, 0, 2767, 886, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3395, 2095] payload_rows_per_dst=[8170, 6830, 8144, 8192, 7286, 6864, 8142, 8104] dup_rows_per_dst=[66, 2, 2, 118, 0, 740, 0, 2876] combine_rows_per_src=[8170, 6830, 8144, 8192, 7286, 6864, 8142, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56776 payload_rows_remote=48736 dup_rows=8760 dup_rows_remote=4312 dedup_ratio=0.133667 router_tokens_per_expert=[0, 0, 0, 338, 0, 0, 93, 1, 0, 0, 0, 0, 0, 0, 3603, 0, 0, 0, 0, 0, 101, 0, 2, 4023, 0, 0, 1, 0, 184, 0, 0, 0, 0, 0, 0, 3608, 3979, 3039, 150, 0, 0, 0, 664, 0, 0, 0, 3891, 3561, 6, 0, 0, 0, 0, 0, 6, 1, 0, 0, 4086, 0, 0, 0, 175, 1256] payload_rows_per_dst=[7474, 8040, 8174, 8192, 6438, 7730, 7976, 2752] dup_rows_per_dst=[606, 4448, 78, 350, 2440, 720, 8, 110] combine_rows_per_src=[7474, 8040, 8174, 8192, 6438, 7730, 7976, 2752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61250 payload_rows_remote=53058 dup_rows=4286 dup_rows_remote=4120 dedup_ratio=0.065399 router_tokens_per_expert=[0, 0, 0, 3, 0, 3, 6, 4, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 0, 131, 0, 0, 4048, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 3959, 4051, 2964, 11, 33, 0, 40, 5, 0, 6, 0, 2944, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 4083, 1701] payload_rows_per_dst=[7932, 7858, 8192, 8046, 7146, 5782, 8102, 8192] dup_rows_per_dst=[18, 0, 166, 0, 226, 500, 0, 3376] combine_rows_per_src=[7932, 7858, 8192, 8046, 7146, 5782, 8102, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=165994496 dispatch_meta_bytes=231560 combine_payload_bytes=165994496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=166226056 ingress_bytes=191160320 link_bytes=191160320 token_bytes=332220552 weight_movement_bytes=25165824 total_nvlink_bytes=357386376 weight_to_token_ratio=0.075750 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.454509162e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=202711040 dispatch_meta_bytes=221688 combine_payload_bytes=202711040 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=202932728 ingress_bytes=253042688 link_bytes=253042688 token_bytes=405643768 weight_movement_bytes=50331648 total_nvlink_bytes=455975416 weight_to_token_ratio=0.124078 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=24576 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.925362481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59366 payload_rows_remote=51182 dup_rows=6170 dup_rows_remote=494 dedup_ratio=0.094147 router_tokens_per_expert=[4, 5, 0, 99, 0, 0, 73, 0, 0, 5, 0, 5, 0, 17, 3881, 0, 0, 9, 15, 0, 1, 0, 28, 4084, 0, 0, 0, 0, 0, 4, 0, 9, 0, 0, 0, 4085, 4048, 3897, 0, 0, 16, 0, 1979, 12, 0, 2, 3415, 1506, 0, 0, 0, 8, 0, 0, 0, 0, 0, 6, 4083, 0, 0, 0, 1447, 25] payload_rows_per_dst=[8180, 7778, 8182, 8186, 7794, 8184, 8112, 2950] dup_rows_per_dst=[352, 38, 92, 6, 0, 5676, 0, 6] combine_rows_per_src=[8180, 7778, 8182, 8186, 7794, 8184, 8112, 2950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=223010816 dispatch_meta_bytes=233000 combine_payload_bytes=223010816 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=223243816 ingress_bytes=323674112 link_bytes=323674112 token_bytes=446254632 weight_movement_bytes=100663296 total_nvlink_bytes=546917928 weight_to_token_ratio=0.225574 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27904 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.462786008e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=199622656 dispatch_meta_bytes=212192 combine_payload_bytes=199622656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199834848 ingress_bytes=224788480 link_bytes=224788480 token_bytes=399457504 weight_movement_bytes=25165824 total_nvlink_bytes=424623328 weight_to_token_ratio=0.063000 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=27008 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.710380604e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60662 payload_rows_remote=52830 dup_rows=4874 dup_rows_remote=4872 dedup_ratio=0.074371 router_tokens_per_expert=[0, 0, 0, 537, 0, 0, 790, 0, 0, 0, 3, 0, 0, 0, 2497, 0, 0, 0, 0, 2, 3, 0, 6, 4095, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 3894, 3984, 2, 1, 0, 0, 302, 0, 1, 1, 3655, 914, 0, 21, 0, 0, 0, 1, 1, 0, 0, 1, 4092, 0, 1, 2, 3673, 192] payload_rows_per_dst=[8190, 5000, 8192, 8186, 7972, 7930, 7832, 7360] dup_rows_per_dst=[2654, 0, 20, 2, 2, 1816, 2, 378] combine_rows_per_src=[8190, 5000, 8192, 8186, 7972, 7930, 7832, 7360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=217325568 dispatch_meta_bytes=228712 combine_payload_bytes=217325568 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=217554280 ingress_bytes=267657216 link_bytes=267657216 token_bytes=434879848 weight_movement_bytes=50331648 total_nvlink_bytes=485211496 weight_to_token_ratio=0.115737 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.036562153e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53422 payload_rows_remote=51478 dup_rows=12114 dup_rows_remote=12114 dedup_ratio=0.184845 router_tokens_per_expert=[0, 1874, 0, 14, 0, 0, 4095, 2, 2841, 123, 0, 0, 4009, 0, 0, 0, 209, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3563, 0, 972, 0, 0, 0, 0, 0, 0, 3787, 3039, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4506, 8192, 8192, 1944, 7068, 7136, 8192, 8192] dup_rows_per_dst=[416, 8138, 418, 0, 3058, 68, 16, 0] combine_rows_per_src=[4506, 8192, 8192, 1944, 7068, 7136, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=209641472 dispatch_meta_bytes=206704 combine_payload_bytes=209641472 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=209848176 ingress_bytes=285138944 link_bytes=285138944 token_bytes=419489648 weight_movement_bytes=75497472 total_nvlink_bytes=494987120 weight_to_token_ratio=0.179975 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.169577904e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=216391680 dispatch_meta_bytes=230808 combine_payload_bytes=216391680 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=216622488 ingress_bytes=266723328 link_bytes=266723328 token_bytes=433014168 weight_movement_bytes=50331648 total_nvlink_bytes=483345816 weight_to_token_ratio=0.116236 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.029456344e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49782 payload_rows_remote=41604 dup_rows=15754 dup_rows_remote=10902 dedup_ratio=0.240387 router_tokens_per_expert=[0, 0, 0, 0, 2264, 190, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4091, 4096, 0, 4088, 0, 4054, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 4061, 0, 0, 0, 0, 0, 0, 101, 0, 0] payload_rows_per_dst=[8178, 8128, 8192, 8188, 8192, 3302, 2948, 2654] dup_rows_per_dst=[4852, 214, 10, 5164, 5400, 6, 0, 108] combine_rows_per_src=[8178, 8128, 8192, 8188, 8192, 3302, 2948, 2654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54364 payload_rows_remote=46176 dup_rows=11172 dup_rows_remote=11150 dedup_ratio=0.170471 router_tokens_per_expert=[0, 330, 0, 2264, 0, 0, 4096, 13, 508, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4094, 1, 0, 0, 4092, 0, 0, 4022, 0, 2514, 0, 0, 0, 0, 1, 0, 4075, 2498, 0, 0, 0, 1, 0, 0, 4, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5106, 1018, 7344, 8192, 8172, 8152, 8192, 8188] dup_rows_per_dst=[108, 0, 0, 2170, 5796, 2764, 312, 22] combine_rows_per_src=[5106, 1018, 7344, 8192, 8172, 8152, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=210853888 dispatch_meta_bytes=254368 combine_payload_bytes=210853888 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=211108256 ingress_bytes=286351360 link_bytes=286351360 token_bytes=421962144 weight_movement_bytes=75497472 total_nvlink_bytes=497459616 weight_to_token_ratio=0.178920 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=17408 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.178802989e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56906 payload_rows_remote=48910 dup_rows=8630 dup_rows_remote=6622 dedup_ratio=0.131683 router_tokens_per_expert=[0, 938, 0, 2125, 0, 0, 4096, 34, 2817, 6, 0, 0, 4090, 0, 0, 0, 5, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 2336, 0, 2731, 0, 0, 0, 0, 0, 0, 3969, 1033, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5374, 5758, 8192, 7984, 7996, 5218, 8192, 8192] dup_rows_per_dst=[820, 460, 14, 5086, 2008, 238, 4, 0] combine_rows_per_src=[5374, 5758, 8192, 7984, 7996, 5218, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61260 payload_rows_remote=53086 dup_rows=4276 dup_rows_remote=3630 dedup_ratio=0.065247 router_tokens_per_expert=[0, 3629, 0, 2, 0, 0, 4096, 0, 7, 317, 0, 0, 4085, 0, 0, 1, 127, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 2927, 0, 0, 0, 0, 0, 0, 1921, 3200, 0, 0, 0, 0, 0, 0, 67, 4, 0, 0, 0, 0, 2, 7, 0, 0, 0, 6, 84, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8068, 8174, 8192, 5854, 6400, 8188, 8192, 8192] dup_rows_per_dst=[3036, 646, 254, 0, 0, 142, 198, 0] combine_rows_per_src=[8068, 8174, 8192, 5854, 6400, 8188, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=170409984 dispatch_meta_bytes=210024 combine_payload_bytes=170409984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=170620008 ingress_bytes=195575808 link_bytes=195575808 token_bytes=341029992 weight_movement_bytes=25165824 total_nvlink_bytes=366195816 weight_to_token_ratio=0.073794 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.488105924e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56064 payload_rows_remote=47874 dup_rows=9472 dup_rows_remote=9300 dedup_ratio=0.144531 router_tokens_per_expert=[0, 894, 0, 1112, 0, 0, 4094, 22, 2584, 33, 0, 0, 4078, 0, 0, 0, 86, 0, 0, 4092, 0, 0, 0, 4095, 0, 0, 3733, 0, 3694, 0, 0, 0, 0, 0, 2, 4077, 134, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8172, 8190, 7388, 272, 7474, 8192, 8184] dup_rows_per_dst=[4018, 5218, 172, 0, 0, 38, 26, 0] combine_rows_per_src=[8192, 8172, 8190, 7388, 272, 7474, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57964 payload_rows_remote=49780 dup_rows=7572 dup_rows_remote=5774 dedup_ratio=0.115540 router_tokens_per_expert=[0, 413, 4, 3403, 16, 0, 4070, 0, 1839, 0, 0, 0, 3528, 0, 12, 0, 13, 20, 0, 4083, 0, 0, 4, 4030, 0, 0, 4060, 0, 2337, 26, 0, 10, 0, 0, 0, 2916, 959, 19, 5, 40, 0, 0, 0, 759, 163, 0, 9, 0, 0, 0, 0, 0, 12, 8, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0] payload_rows_per_dst=[7162, 3702, 8118, 7710, 6742, 8184, 8162, 8184] dup_rows_per_dst=[510, 0, 16, 4092, 1136, 1798, 18, 2] combine_rows_per_src=[7162, 3702, 8118, 7710, 6742, 8184, 8162, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=189136896 dispatch_meta_bytes=229304 combine_payload_bytes=189136896 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=189366200 ingress_bytes=239468544 link_bytes=239468544 token_bytes=378503096 weight_movement_bytes=50331648 total_nvlink_bytes=428834744 weight_to_token_ratio=0.132976 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.822078929e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=200335360 dispatch_meta_bytes=222128 combine_payload_bytes=200335360 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=200557488 ingress_bytes=275832832 link_bytes=275832832 token_bytes=400892848 weight_movement_bytes=75497472 total_nvlink_bytes=476390320 weight_to_token_ratio=0.188323 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.098769144e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64244 payload_rows_remote=56742 dup_rows=1292 dup_rows_remote=1292 dedup_ratio=0.019714 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 203, 0, 0, 3859, 1, 0, 0, 0, 4096, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 96, 0, 4096, 0, 0, 0, 0, 3948, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7886, 7502, 8192, 8192, 7896, 8192] dup_rows_per_dst=[0, 858, 240, 0, 0, 194, 0, 0] combine_rows_per_src=[8192, 8192, 7886, 7502, 8192, 8192, 7896, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56266 payload_rows_remote=48076 dup_rows=9270 dup_rows_remote=9256 dedup_ratio=0.141449 router_tokens_per_expert=[0, 307, 1, 2324, 1, 0, 4094, 1, 1660, 2, 0, 0, 4014, 2, 0, 1, 0, 0, 0, 4094, 0, 0, 1, 4095, 0, 0, 3803, 0, 2778, 0, 0, 0, 1, 0, 0, 3359, 1597, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 0, 0, 4, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4886, 3324, 8192, 8036, 7842, 7608, 8190, 8188] dup_rows_per_dst=[382, 6, 0, 5548, 2072, 1248, 14, 0] combine_rows_per_src=[4886, 3324, 8192, 8036, 7842, 7608, 8190, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=196091904 dispatch_meta_bytes=228696 combine_payload_bytes=196091904 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=196320600 ingress_bytes=271589376 link_bytes=271589376 token_bytes=392412504 weight_movement_bytes=75497472 total_nvlink_bytes=467909976 weight_to_token_ratio=0.192393 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.066481347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=217440256 dispatch_meta_bytes=226864 combine_payload_bytes=217440256 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=217667120 ingress_bytes=267771904 link_bytes=267771904 token_bytes=435107376 weight_movement_bytes=50331648 total_nvlink_bytes=485439024 weight_to_token_ratio=0.115676 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.037434796e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=203898880 dispatch_meta_bytes=222216 combine_payload_bytes=203898880 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=204121096 ingress_bytes=254230528 link_bytes=254230528 token_bytes=408019976 weight_movement_bytes=50331648 total_nvlink_bytes=458351624 weight_to_token_ratio=0.123356 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.934400571e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232415232 dispatch_meta_bytes=232136 combine_payload_bytes=232415232 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=232647368 ingress_bytes=282746880 link_bytes=282746880 token_bytes=465062600 weight_movement_bytes=50331648 total_nvlink_bytes=515394248 weight_to_token_ratio=0.108226 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.151377061e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56002 payload_rows_remote=47922 dup_rows=9534 dup_rows_remote=8058 dedup_ratio=0.145477 router_tokens_per_expert=[0, 0, 0, 798, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 4091, 3008, 0, 0, 0, 0, 607, 0, 0, 0, 3151, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 3635, 1559] payload_rows_per_dst=[8080, 7924, 8190, 8128, 6016, 1484, 8182, 7998] dup_rows_per_dst=[1476, 5650, 0, 0, 0, 18, 0, 2390] combine_rows_per_src=[8080, 7924, 8190, 8128, 6016, 1484, 8182, 7998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=196919296 dispatch_meta_bytes=229328 combine_payload_bytes=196919296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197148624 ingress_bytes=222085120 link_bytes=222085120 token_bytes=394067920 weight_movement_bytes=25165824 total_nvlink_bytes=419233744 weight_to_token_ratio=0.063862 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.689811158e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54514 payload_rows_remote=46322 dup_rows=11022 dup_rows_remote=11018 dedup_ratio=0.168182 router_tokens_per_expert=[0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 130, 0, 0, 2595, 78, 0, 3497, 0, 0, 0, 1, 4087, 0, 0, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 568, 0, 4096, 1, 0, 0, 0, 3828, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7996, 262, 5320, 8192, 8182, 8192, 8178, 8192] dup_rows_per_dst=[20, 0, 28, 3160, 0, 1142, 6668, 4] combine_rows_per_src=[7996, 262, 5320, 8192, 8182, 8192, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57084 payload_rows_remote=48914 dup_rows=8452 dup_rows_remote=8452 dedup_ratio=0.128967 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 9, 1094, 0, 0, 2397, 26, 0, 3047, 1, 0, 0, 0, 4088, 0, 0, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1305, 0, 4096, 0, 0, 0, 0, 2679, 0, 4085, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7300, 3488, 8170, 8192, 5358, 8192] dup_rows_per_dst=[10, 2190, 3642, 0, 0, 2610, 0, 0] combine_rows_per_src=[8192, 8192, 7300, 3488, 8170, 8192, 5358, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=196288512 dispatch_meta_bytes=223920 combine_payload_bytes=196288512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=196512432 ingress_bytes=221454336 link_bytes=221454336 token_bytes=392800944 weight_movement_bytes=25165824 total_nvlink_bytes=417966768 weight_to_token_ratio=0.064068 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.685011621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56920 payload_rows_remote=51552 dup_rows=8616 dup_rows_remote=8344 dedup_ratio=0.131470 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 37, 1011, 0, 0, 835, 341, 0, 1642, 0, 0, 0, 2, 4070, 0, 0, 2905, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2278, 0, 4096, 0, 0, 0, 0, 3284, 0, 4011, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 5368, 6424, 8026, 5960, 6568, 8190] dup_rows_per_dst=[126, 2044, 272, 3028, 0, 3146, 0, 0] combine_rows_per_src=[8192, 8192, 5368, 6424, 8026, 5960, 6568, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54066 payload_rows_remote=45874 dup_rows=11470 dup_rows_remote=7912 dedup_ratio=0.175018 router_tokens_per_expert=[0, 0, 0, 0, 0, 50, 24, 0, 0, 0, 0, 0, 2, 1777, 0, 0, 3849, 1, 0, 3784, 8, 0, 0, 6, 4096, 0, 0, 127, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 5, 163, 0, 4095, 0, 0, 0, 0, 2292, 0, 4033, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8188, 8190, 8192, 336, 4584, 8192] dup_rows_per_dst=[134, 3558, 7108, 254, 416, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8188, 8190, 8192, 336, 4584, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60318 payload_rows_remote=52164 dup_rows=5218 dup_rows_remote=654 dedup_ratio=0.079620 router_tokens_per_expert=[0, 0, 0, 0, 44, 2, 0, 0, 0, 0, 0, 45, 0, 74, 0, 2, 1396, 226, 14, 1729, 0, 0, 0, 0, 4071, 0, 0, 3762, 2, 0, 0, 0, 0, 12, 0, 0, 12, 0, 0, 0, 0, 1020, 0, 4048, 0, 20, 21, 1, 4064, 0, 3956, 70, 19, 6, 0, 4054, 4039, 0, 0, 0, 9, 16, 14, 20] payload_rows_per_dst=[8156, 8156, 4082, 7528, 7960, 8154, 8164, 8118] dup_rows_per_dst=[44, 228, 150, 0, 0, 4564, 154, 78] combine_rows_per_src=[8156, 8156, 4082, 7528, 7960, 8154, 8164, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=189734912 dispatch_meta_bytes=229360 combine_payload_bytes=189734912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189964272 ingress_bytes=214900736 link_bytes=214900736 token_bytes=379699184 weight_movement_bytes=25165824 total_nvlink_bytes=404865008 weight_to_token_ratio=0.066278 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.635146297e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61832 payload_rows_remote=56108 dup_rows=3704 dup_rows_remote=3296 dedup_ratio=0.056519 router_tokens_per_expert=[0, 725, 0, 0, 0, 245, 0, 0, 42, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4021, 4096, 2542, 0, 0, 0, 0, 735, 0, 0, 1, 1, 0, 0, 0, 15, 0, 0, 0, 0, 3865, 0, 0, 0, 4096, 0, 1102, 3407, 0, 0, 3648, 0, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8046, 8166, 8192, 5724, 8192, 7746, 7576, 8190] dup_rows_per_dst=[1190, 218, 6, 408, 34, 406, 1442, 0] combine_rows_per_src=[8046, 8166, 8192, 5724, 8192, 7746, 7576, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=200351744 dispatch_meta_bytes=229464 combine_payload_bytes=200351744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200581208 ingress_bytes=225517568 link_bytes=225517568 token_bytes=400932952 weight_movement_bytes=25165824 total_nvlink_bytes=426098776 weight_to_token_ratio=0.062768 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.715928121e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=211156992 dispatch_meta_bytes=239584 combine_payload_bytes=211156992 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=211396576 ingress_bytes=286654464 link_bytes=286654464 token_bytes=422553568 weight_movement_bytes=75497472 total_nvlink_bytes=498051040 weight_to_token_ratio=0.178670 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=16640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.181109260e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=187899904 dispatch_meta_bytes=215144 combine_payload_bytes=187899904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188115048 ingress_bytes=213065728 link_bytes=213065728 token_bytes=376014952 weight_movement_bytes=25165824 total_nvlink_bytes=401180776 weight_to_token_ratio=0.066928 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.621184006e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=213663744 dispatch_meta_bytes=211272 combine_payload_bytes=213663744 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=213875016 ingress_bytes=263995392 link_bytes=263995392 token_bytes=427538760 weight_movement_bytes=50331648 total_nvlink_bytes=477870408 weight_to_token_ratio=0.117724 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22784 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.008699903e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58188 payload_rows_remote=50126 dup_rows=7348 dup_rows_remote=5346 dedup_ratio=0.112122 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 2074, 0, 0, 2356, 5, 0, 0, 0, 4095, 0, 0, 3274, 0, 0, 0, 2, 0, 5, 0, 7, 0, 0, 0, 0, 0, 482, 0, 4095, 0, 0, 0, 0, 4028, 0, 4095, 0, 1, 1, 0, 4045, 4091, 0, 0, 0, 1, 1, 0, 0] payload_rows_per_dst=[8092, 6454, 4158, 6856, 8192, 8190, 8062, 8184] dup_rows_per_dst=[2, 204, 0, 1444, 22, 3672, 2002, 2] combine_rows_per_src=[8092, 6454, 4158, 6856, 8192, 8190, 8062, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229818368 dispatch_meta_bytes=237616 combine_payload_bytes=229818368 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=230055984 ingress_bytes=330481664 link_bytes=330481664 token_bytes=459874352 weight_movement_bytes=100663296 total_nvlink_bytes=560537648 weight_to_token_ratio=0.218893 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=13824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.514583613e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56152 payload_rows_remote=47960 dup_rows=9384 dup_rows_remote=4624 dedup_ratio=0.143188 router_tokens_per_expert=[0, 1206, 0, 896, 0, 0, 4096, 3, 2277, 9, 0, 0, 4078, 0, 0, 0, 25, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3622, 0, 766, 0, 0, 0, 0, 0, 0, 4091, 3487, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 1532, 6974, 7250, 7636, 8192] dup_rows_per_dst=[4760, 4544, 50, 0, 0, 30, 0, 0] combine_rows_per_src=[8192, 8184, 8192, 1532, 6974, 7250, 7636, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=205316096 dispatch_meta_bytes=221888 combine_payload_bytes=205316096 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=205537984 ingress_bytes=280813568 link_bytes=280813568 token_bytes=410854080 weight_movement_bytes=75497472 total_nvlink_bytes=486351552 weight_to_token_ratio=0.183757 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=28160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.136666790e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49188 payload_rows_remote=41058 dup_rows=16348 dup_rows_remote=16348 dedup_ratio=0.249451 router_tokens_per_expert=[0, 707, 1, 0, 0, 367, 0, 0, 733, 3, 0, 29, 1, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 4083, 4096, 2732, 0, 0, 0, 0, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 0, 0, 0, 4096, 0, 929, 2235, 0, 0, 3909, 0, 0, 4060, 0, 5, 0, 0, 0, 0] payload_rows_per_dst=[1720, 1476, 7734, 8192, 6992, 6810, 8134, 8130] dup_rows_per_dst=[430, 56, 176, 6708, 0, 1766, 7212, 0] combine_rows_per_src=[1720, 1476, 7734, 8192, 6992, 6810, 8134, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54568 payload_rows_remote=46380 dup_rows=10968 dup_rows_remote=10942 dedup_ratio=0.167358 router_tokens_per_expert=[0, 964, 0, 0, 0, 855, 0, 0, 1249, 232, 0, 16, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 3871, 4096, 4025, 0, 0, 0, 0, 1422, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2498, 0, 0, 0, 4092, 0, 966, 188, 0, 0, 4090, 0, 0, 4096, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[4624, 7874, 8192, 3246, 8188, 8190, 6062, 8192] dup_rows_per_dst=[906, 2862, 180, 602, 26, 4856, 1530, 6] combine_rows_per_src=[4624, 7874, 8192, 3246, 8188, 8190, 6062, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=196444160 dispatch_meta_bytes=210336 combine_payload_bytes=196444160 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=196654496 ingress_bytes=246775808 link_bytes=246775808 token_bytes=393098656 weight_movement_bytes=50331648 total_nvlink_bytes=443430304 weight_to_token_ratio=0.128038 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=12160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.877678765e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54030 payload_rows_remote=45852 dup_rows=11506 dup_rows_remote=7974 dedup_ratio=0.175568 router_tokens_per_expert=[0, 3692, 8, 0, 0, 7, 27, 0, 926, 1146, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3781, 4096, 550, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 4001, 0, 0, 0, 4096, 0, 14, 1156, 0, 0, 4085, 0, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 8192, 2748, 8192, 8002, 2340, 8186] dup_rows_per_dst=[7446, 3532, 24, 60, 444, 0, 0, 0] combine_rows_per_src=[8192, 8178, 8192, 2748, 8192, 8002, 2340, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51028 payload_rows_remote=42836 dup_rows=14508 dup_rows_remote=14408 dedup_ratio=0.221375 router_tokens_per_expert=[0, 1839, 10, 0, 0, 2648, 12, 0, 358, 273, 1, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 3624, 4096, 3489, 0, 0, 0, 0, 2504, 0, 0, 245, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1195, 0, 0, 0, 4096, 0, 81, 129, 0, 0, 3958, 0, 0, 4079, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7384, 8192, 8100, 8192, 2390, 420, 8158] dup_rows_per_dst=[8742, 1128, 100, 3886, 636, 0, 0, 16] combine_rows_per_src=[8192, 7384, 8192, 8100, 8192, 2390, 420, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=168173568 dispatch_meta_bytes=229624 combine_payload_bytes=168173568 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=168403192 ingress_bytes=193339392 link_bytes=193339392 token_bytes=336576760 weight_movement_bytes=25165824 total_nvlink_bytes=361742584 weight_to_token_ratio=0.074770 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.471089382e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52522 payload_rows_remote=44334 dup_rows=13014 dup_rows_remote=7456 dedup_ratio=0.198578 router_tokens_per_expert=[0, 0, 3, 0, 0, 171, 0, 14, 489, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 31, 0, 0, 9, 4072, 4023, 4072, 0, 0, 5, 52, 1549, 11, 0, 1, 0, 0, 0, 0, 0, 0, 23, 23, 17, 2735, 0, 0, 3, 4073, 0, 10, 2695, 0, 0, 4052, 4, 14, 3806, 4, 0, 425, 9, 119, 2] payload_rows_per_dst=[376, 8160, 8126, 3212, 8168, 8188, 8112, 8180] dup_rows_per_dst=[0, 1438, 28, 0, 2, 5558, 5410, 578] combine_rows_per_src=[376, 8160, 8126, 3212, 8168, 8188, 8112, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64220 payload_rows_remote=56690 dup_rows=1316 dup_rows_remote=1312 dedup_ratio=0.020081 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4092, 3956, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 87, 0, 0, 4079, 372, 0, 0, 3756, 0, 9, 0, 0, 0, 2, 0, 205, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0] payload_rows_per_dst=[7912, 8192, 8192, 7530, 8166, 7852, 8184, 8192] dup_rows_per_dst=[0, 0, 886, 4, 410, 0, 0, 16] combine_rows_per_src=[7912, 8192, 8192, 7530, 8166, 7852, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=189972480 dispatch_meta_bytes=229288 combine_payload_bytes=189972480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=190201768 ingress_bytes=215138304 link_bytes=215138304 token_bytes=380174248 weight_movement_bytes=25165824 total_nvlink_bytes=405340072 weight_to_token_ratio=0.066195 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.636953915e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=187809792 dispatch_meta_bytes=215304 combine_payload_bytes=187809792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188025096 ingress_bytes=212975616 link_bytes=212975616 token_bytes=375834888 weight_movement_bytes=25165824 total_nvlink_bytes=401000712 weight_to_token_ratio=0.066960 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.620498358e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=175456256 dispatch_meta_bytes=228976 combine_payload_bytes=175456256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=175685232 ingress_bytes=200622080 link_bytes=200622080 token_bytes=351141488 weight_movement_bytes=25165824 total_nvlink_bytes=376307312 weight_to_token_ratio=0.071669 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.526502223e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=181592064 dispatch_meta_bytes=207160 combine_payload_bytes=181592064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=181799224 ingress_bytes=206757888 link_bytes=206757888 token_bytes=363391288 weight_movement_bytes=25165824 total_nvlink_bytes=388557112 weight_to_token_ratio=0.069253 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=19072 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.573188632e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54454 payload_rows_remote=46262 dup_rows=11082 dup_rows_remote=4414 dedup_ratio=0.169098 router_tokens_per_expert=[0, 2045, 0, 0, 0, 1396, 0, 1, 737, 200, 0, 12, 0, 2, 0, 0, 0, 0, 1, 7, 0, 0, 3, 0, 4089, 4088, 2714, 0, 0, 0, 0, 1536, 1, 0, 5, 0, 0, 0, 0, 3, 0, 0, 1, 1, 336, 0, 0, 1, 4094, 0, 171, 3328, 0, 0, 3930, 1, 0, 4050, 0, 0, 8, 1, 0, 6] payload_rows_per_dst=[5718, 2352, 8182, 7894, 8192, 5798, 8192, 8126] dup_rows_per_dst=[1166, 346, 16, 2560, 14, 308, 6668, 4] combine_rows_per_src=[5718, 2352, 8182, 7894, 8192, 5798, 8192, 8126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232202240 dispatch_meta_bytes=232008 combine_payload_bytes=232202240 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=232434248 ingress_bytes=332865536 link_bytes=332865536 token_bytes=464636488 weight_movement_bytes=100663296 total_nvlink_bytes=565299784 weight_to_token_ratio=0.216650 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=29312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.532722124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59836 payload_rows_remote=51648 dup_rows=5700 dup_rows_remote=5700 dedup_ratio=0.086975 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 308, 0, 0, 385, 1762, 0, 3750, 0, 0, 0, 0, 4096, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 0, 4096, 0, 0, 0, 0, 3951, 0, 4092, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 7994, 8192, 8184, 2992, 7902, 8192] dup_rows_per_dst=[0, 622, 3800, 1278, 0, 0, 0, 0] combine_rows_per_src=[8188, 8192, 7994, 8192, 8184, 2992, 7902, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62008 payload_rows_remote=53818 dup_rows=3528 dup_rows_remote=3528 dedup_ratio=0.053833 router_tokens_per_expert=[1, 0, 0, 0, 4095, 0, 4046, 4071, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 46, 0, 0, 4074, 19, 0, 0, 4070, 0, 87, 0, 0, 0, 5, 0, 73, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8028, 7982, 8190, 5236, 8106, 8154, 8122, 8190] dup_rows_per_dst=[2, 2, 88, 110, 2070, 1140, 116, 0] combine_rows_per_src=[8028, 7982, 8190, 5236, 8106, 8154, 8122, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=189489152 dispatch_meta_bytes=202704 combine_payload_bytes=189489152 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=189691856 ingress_bytes=290152448 link_bytes=290152448 token_bytes=379181008 weight_movement_bytes=100663296 total_nvlink_bytes=479844304 weight_to_token_ratio=0.265476 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=14592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.207724877e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=211550208 dispatch_meta_bytes=229392 combine_payload_bytes=211550208 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=211779600 ingress_bytes=236716032 link_bytes=236716032 token_bytes=423329808 weight_movement_bytes=25165824 total_nvlink_bytes=448495632 weight_to_token_ratio=0.059447 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.801135493e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64224 payload_rows_remote=56704 dup_rows=1312 dup_rows_remote=1302 dedup_ratio=0.020020 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4093, 3874, 0, 0, 4096, 0, 1, 0, 5, 0, 0, 71, 0, 0, 4096, 244, 0, 0, 3992, 0, 350, 0, 0, 0, 6, 0, 31, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[7748, 8192, 8192, 8044, 7520, 8150, 8186, 8192] dup_rows_per_dst=[0, 12, 630, 652, 10, 0, 0, 8] combine_rows_per_src=[7748, 8192, 8192, 8044, 7520, 8150, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60194 payload_rows_remote=52002 dup_rows=5342 dup_rows_remote=5140 dedup_ratio=0.081512 router_tokens_per_expert=[16, 0, 0, 0, 4065, 0, 3676, 2501, 0, 0, 4096, 0, 97, 0, 4, 0, 0, 1229, 0, 2, 3990, 0, 0, 0, 3988, 0, 0, 0, 8, 0, 3, 0, 1244, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 208, 0] payload_rows_per_dst=[5034, 8192, 8192, 7976, 8192, 7078, 7366, 8164] dup_rows_per_dst=[0, 202, 2250, 22, 2486, 0, 0, 382] combine_rows_per_src=[5034, 8192, 8192, 7976, 8192, 7078, 7366, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=220438528 dispatch_meta_bytes=229384 combine_payload_bytes=220438528 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=220667912 ingress_bytes=245604352 link_bytes=245604352 token_bytes=441106440 weight_movement_bytes=25165824 total_nvlink_bytes=466272264 weight_to_token_ratio=0.057052 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.868765339e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64640 payload_rows_remote=56448 dup_rows=896 dup_rows_remote=556 dedup_ratio=0.013672 router_tokens_per_expert=[2, 0, 0, 0, 4089, 0, 4096, 4096, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 172, 0, 0, 4078, 16, 0, 0, 3818, 0, 81, 0, 1, 0, 19, 0, 221, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8192, 8192, 8192, 7728, 8060, 7904, 8192, 8180] dup_rows_per_dst=[4, 8, 340, 110, 434, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7728, 8060, 7904, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59406 payload_rows_remote=51238 dup_rows=6130 dup_rows_remote=5430 dedup_ratio=0.093536 router_tokens_per_expert=[0, 0, 0, 33, 3985, 0, 3999, 3417, 0, 0, 4045, 10, 19, 16, 0, 0, 0, 276, 11, 0, 4082, 129, 1, 5, 4065, 0, 1748, 0, 42, 14, 1, 19, 0, 1814, 4, 0, 1, 0, 0, 79, 0, 0, 0, 4077, 0, 0, 19, 338, 0, 12, 23, 0, 19, 0, 4, 454, 0, 7, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6900, 8138, 8182, 8184, 3796, 8168, 8054, 7984] dup_rows_per_dst=[0, 42, 826, 3594, 0, 700, 968, 0] combine_rows_per_src=[6900, 8138, 8182, 8184, 3796, 8168, 8054, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58594 payload_rows_remote=50404 dup_rows=6942 dup_rows_remote=2756 dedup_ratio=0.105927 router_tokens_per_expert=[5, 1576, 0, 173, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3534, 0, 0, 0, 0, 2048, 0, 0, 0, 0, 45, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1805, 0, 2650, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 4095, 0, 3665, 0] payload_rows_per_dst=[4596, 8192, 7068, 8190, 8192, 6508, 8192, 7656] dup_rows_per_dst=[966, 272, 0, 4186, 1168, 348, 0, 2] combine_rows_per_src=[4596, 8192, 7068, 8190, 8192, 6508, 8192, 7656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=232259584 dispatch_meta_bytes=232024 combine_payload_bytes=232259584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=232491608 ingress_bytes=257425408 link_bytes=257425408 token_bytes=464751192 weight_movement_bytes=25165824 total_nvlink_bytes=489917016 weight_to_token_ratio=0.054149 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.958709916e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=213000192 dispatch_meta_bytes=228568 combine_payload_bytes=213000192 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=213228760 ingress_bytes=238166016 link_bytes=238166016 token_bytes=426228952 weight_movement_bytes=25165824 total_nvlink_bytes=451394776 weight_to_token_ratio=0.059043 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.812168196e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=209870848 dispatch_meta_bytes=226672 combine_payload_bytes=209870848 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210097520 ingress_bytes=235036672 link_bytes=235036672 token_bytes=419968368 weight_movement_bytes=25165824 total_nvlink_bytes=445134192 weight_to_token_ratio=0.059923 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788357504e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=231211008 dispatch_meta_bytes=228016 combine_payload_bytes=231211008 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=231439024 ingress_bytes=281542656 link_bytes=281542656 token_bytes=462650032 weight_movement_bytes=50331648 total_nvlink_bytes=512981680 weight_to_token_ratio=0.108790 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.142214308e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65298 payload_rows_remote=57148 dup_rows=238 dup_rows_remote=232 dedup_ratio=0.003632 router_tokens_per_expert=[0, 0, 0, 0, 4091, 0, 4073, 4001, 0, 1, 4093, 0, 0, 1, 0, 0, 5, 2, 1, 0, 4094, 0, 0, 0, 4095, 0, 5, 0, 0, 1, 0, 0, 76, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 32, 0] payload_rows_per_dst=[8002, 8188, 8192, 8192, 8192, 8192, 8150, 8190] dup_rows_per_dst=[0, 2, 12, 10, 150, 2, 6, 56] combine_rows_per_src=[8002, 8188, 8192, 8192, 8192, 8192, 8150, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=206454784 dispatch_meta_bytes=212640 combine_payload_bytes=206454784 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206667424 ingress_bytes=231620608 link_bytes=231620608 token_bytes=413122208 weight_movement_bytes=25165824 total_nvlink_bytes=438288032 weight_to_token_ratio=0.060916 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17152 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.762365204e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56582 payload_rows_remote=48390 dup_rows=8954 dup_rows_remote=8358 dedup_ratio=0.136627 router_tokens_per_expert=[0, 104, 0, 0, 0, 194, 0, 0, 2002, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 4096, 4095, 0, 0, 0, 0, 2003, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 0, 4096, 0, 650, 365, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8192, 5518, 2338, 7770] dup_rows_per_dst=[596, 4176, 4, 4006, 72, 0, 100, 0] combine_rows_per_src=[8192, 8190, 8192, 8190, 8192, 5518, 2338, 7770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56500 payload_rows_remote=48428 dup_rows=9036 dup_rows_remote=5208 dedup_ratio=0.137878 router_tokens_per_expert=[2, 4068, 0, 971, 0, 0, 0, 0, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 3086, 0, 0, 0, 0, 2708, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 686, 0, 3194, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3991, 0, 53, 0] payload_rows_per_dst=[8136, 3552, 7918, 5436, 8192, 7002, 8192, 8072] dup_rows_per_dst=[1946, 102, 2208, 4, 18, 758, 172, 3828] combine_rows_per_src=[8136, 3552, 7918, 5436, 8192, 7002, 8192, 8072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=234078208 dispatch_meta_bytes=229520 combine_payload_bytes=234078208 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=234307728 ingress_bytes=284409856 link_bytes=284409856 token_bytes=468385936 weight_movement_bytes=50331648 total_nvlink_bytes=518717584 weight_to_token_ratio=0.107458 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.164030387e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=198205440 dispatch_meta_bytes=226992 combine_payload_bytes=198205440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198432432 ingress_bytes=223371264 link_bytes=223371264 token_bytes=396637872 weight_movement_bytes=25165824 total_nvlink_bytes=421803696 weight_to_token_ratio=0.063448 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.699597228e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55274 payload_rows_remote=47082 dup_rows=10262 dup_rows_remote=7446 dedup_ratio=0.156586 router_tokens_per_expert=[157, 3653, 0, 0, 0, 0, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3181, 0, 4, 0, 0, 18, 0, 0, 0, 0, 32, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 495, 0, 3579, 0, 0, 4096, 0, 11, 0, 0, 2, 0, 0, 0, 23, 3, 0, 4086, 0, 3827, 0] payload_rows_per_dst=[8190, 8192, 6370, 8172, 8192, 7914, 8192, 52] dup_rows_per_dst=[6588, 2816, 0, 100, 2, 730, 26, 0] combine_rows_per_src=[8190, 8192, 6370, 8172, 8192, 7914, 8192, 52] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56500 payload_rows_remote=48308 dup_rows=9036 dup_rows_remote=9034 dedup_ratio=0.137878 router_tokens_per_expert=[19, 526, 0, 194, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3844, 0, 0, 0, 0, 3198, 0, 0, 0, 0, 93, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 0, 3636, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 79, 0, 4083, 0, 3097, 0] payload_rows_per_dst=[1466, 8192, 7688, 6948, 8192, 7796, 8192, 8026] dup_rows_per_dst=[12, 858, 0, 226, 2, 2034, 4, 5900] combine_rows_per_src=[1466, 8192, 7688, 6948, 8192, 7796, 8192, 8026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=198361088 dispatch_meta_bytes=214544 combine_payload_bytes=198361088 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=198575632 ingress_bytes=273858560 link_bytes=273858560 token_bytes=396936720 weight_movement_bytes=75497472 total_nvlink_bytes=472434192 weight_to_token_ratio=0.190200 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.083747216e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56752 payload_rows_remote=48668 dup_rows=8784 dup_rows_remote=6488 dedup_ratio=0.134033 router_tokens_per_expert=[0, 374, 11, 1401, 0, 15, 0, 0, 18, 178, 5, 0, 8, 0, 0, 14, 0, 4057, 0, 3747, 4070, 0, 299, 0, 13, 2346, 0, 19, 0, 0, 567, 0, 10, 0, 0, 0, 0, 70, 0, 0, 29, 0, 0, 1134, 0, 3973, 31, 23, 3989, 0, 0, 0, 0, 0, 0, 2, 0, 511, 0, 0, 2871, 0, 2974, 9] payload_rows_per_dst=[3294, 7600, 8154, 5546, 8134, 8084, 7982, 7958] dup_rows_per_dst=[308, 340, 584, 344, 140, 2296, 0, 4772] combine_rows_per_src=[3294, 7600, 8154, 5546, 8134, 8084, 7982, 7958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60626 payload_rows_remote=53044 dup_rows=4910 dup_rows_remote=4910 dedup_ratio=0.074921 router_tokens_per_expert=[5, 1082, 0, 157, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3791, 0, 0, 0, 0, 1802, 0, 0, 0, 19, 46, 0, 0, 66, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 0, 3714, 0, 0, 4078, 0, 24, 0, 0, 0, 0, 0, 0, 23, 348, 0, 2706, 0, 3398, 0] payload_rows_per_dst=[7954, 8192, 7582, 7104, 8192, 6226, 8192, 7184] dup_rows_per_dst=[1962, 406, 0, 2042, 134, 180, 12, 174] combine_rows_per_src=[7954, 8192, 7582, 7104, 8192, 6226, 8192, 7184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63756 payload_rows_remote=55634 dup_rows=1780 dup_rows_remote=1780 dedup_ratio=0.027161 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 4096, 0, 0, 4061, 4061, 0, 0, 0, 0, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 0, 0, 2, 0, 584, 4096, 3988, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7074, 8154, 8192, 8122, 7726, 8148, 8148, 8192] dup_rows_per_dst=[0, 0, 738, 0, 0, 46, 996, 0] combine_rows_per_src=[7074, 8154, 8192, 8122, 7726, 8148, 8148, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=192847872 dispatch_meta_bytes=218112 combine_payload_bytes=192847872 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193065984 ingress_bytes=218013696 link_bytes=218013696 token_bytes=385913856 weight_movement_bytes=25165824 total_nvlink_bytes=411079680 weight_to_token_ratio=0.065211 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=28544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.658832326e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=197869568 dispatch_meta_bytes=229368 combine_payload_bytes=197869568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198098936 ingress_bytes=223035392 link_bytes=223035392 token_bytes=395968504 weight_movement_bytes=25165824 total_nvlink_bytes=421134328 weight_to_token_ratio=0.063555 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.697041630e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=199344128 dispatch_meta_bytes=220624 combine_payload_bytes=199344128 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=199564752 ingress_bytes=249675776 link_bytes=249675776 token_bytes=398908880 weight_movement_bytes=50331648 total_nvlink_bytes=449240528 weight_to_token_ratio=0.126173 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.899744171e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217268224 dispatch_meta_bytes=231816 combine_payload_bytes=217268224 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=217500040 ingress_bytes=317931520 link_bytes=317931520 token_bytes=434768264 weight_movement_bytes=100663296 total_nvlink_bytes=535431560 weight_to_token_ratio=0.231533 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.419091518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60354 payload_rows_remote=52170 dup_rows=5182 dup_rows_remote=5178 dedup_ratio=0.079071 router_tokens_per_expert=[0, 2678, 1, 396, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 1, 1, 0, 4092, 0, 4095, 4088, 0, 0, 0, 0, 3845, 0, 1, 0, 0, 2, 0, 0, 19, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 0, 3319, 1, 0, 4088, 4, 0, 0, 1, 0, 0, 1, 0, 12, 0, 2, 2779, 0, 3148, 1] payload_rows_per_dst=[5586, 8192, 8176, 7694, 8184, 6652, 8184, 7686] dup_rows_per_dst=[564, 352, 0, 2, 38, 24, 4, 4198] combine_rows_per_src=[5586, 8192, 8176, 7694, 8184, 6652, 8184, 7686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=227876864 dispatch_meta_bytes=229656 combine_payload_bytes=227876864 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=228106520 ingress_bytes=303374336 link_bytes=303374336 token_bytes=455983384 weight_movement_bytes=75497472 total_nvlink_bytes=531480856 weight_to_token_ratio=0.165571 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=27136 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.308328167e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58796 payload_rows_remote=50632 dup_rows=6740 dup_rows_remote=1190 dedup_ratio=0.102844 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4096, 3612, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 173, 0, 0, 4096, 114, 0, 0, 3406, 0, 1003, 0, 0, 0, 0, 0, 85, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[8164, 8192, 2862, 7426, 7786, 8192, 8192, 7982] dup_rows_per_dst=[5550, 0, 310, 718, 158, 4, 0, 0] combine_rows_per_src=[8164, 8192, 2862, 7426, 7786, 8192, 8192, 7982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53384 payload_rows_remote=45196 dup_rows=12152 dup_rows_remote=12096 dedup_ratio=0.185425 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4041, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 4096, 0, 0, 3597, 1808, 5, 0, 0, 0, 0, 2431, 0, 0, 0, 0, 0, 0, 0, 120, 0, 3694, 0, 0, 0, 0, 298, 4094, 4068, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8086, 8100, 928, 3626, 8192, 8170, 8094, 8188] dup_rows_per_dst=[0, 0, 0, 0, 4862, 6652, 582, 56] combine_rows_per_src=[8086, 8100, 928, 3626, 8192, 8170, 8094, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=213688320 dispatch_meta_bytes=229392 combine_payload_bytes=213688320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213917712 ingress_bytes=238854144 link_bytes=238854144 token_bytes=427606032 weight_movement_bytes=25165824 total_nvlink_bytes=452771856 weight_to_token_ratio=0.058853 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.817404055e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=207388672 dispatch_meta_bytes=207288 combine_payload_bytes=207388672 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=207595960 ingress_bytes=308051968 link_bytes=308051968 token_bytes=414984632 weight_movement_bytes=100663296 total_nvlink_bytes=515647928 weight_to_token_ratio=0.242571 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.343919542e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61632 payload_rows_remote=53636 dup_rows=3904 dup_rows_remote=3904 dedup_ratio=0.059570 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3810, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 4096, 0, 0, 3307, 3871, 5, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 64, 219, 0, 86, 0, 0, 131, 0, 7, 4096, 4096, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7842, 7996, 8192, 8000, 8168, 5050, 8192, 8192] dup_rows_per_dst=[0, 0, 1574, 1926, 0, 390, 14, 0] combine_rows_per_src=[7842, 7996, 8192, 8000, 8168, 5050, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=185122816 dispatch_meta_bytes=229168 combine_payload_bytes=185122816 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=185351984 ingress_bytes=235454464 link_bytes=235454464 token_bytes=370474800 weight_movement_bytes=50331648 total_nvlink_bytes=420806448 weight_to_token_ratio=0.135857 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791536418e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63846 payload_rows_remote=55786 dup_rows=1690 dup_rows_remote=1688 dedup_ratio=0.025787 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4096, 0, 0, 4033, 3389, 6, 1, 0, 0, 0, 4030, 0, 0, 0, 1, 0, 0, 0, 0, 0, 144, 0, 0, 4, 0, 844, 4096, 3905, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8158, 8186, 8192, 6790, 8060, 8168, 8100, 8192] dup_rows_per_dst=[0, 0, 92, 2, 2, 194, 1398, 2] combine_rows_per_src=[8158, 8186, 8192, 6790, 8060, 8168, 8100, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59070 payload_rows_remote=51002 dup_rows=6466 dup_rows_remote=4696 dedup_ratio=0.098663 router_tokens_per_expert=[13, 3, 0, 120, 0, 27, 4079, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 10, 0, 1414, 0, 4069, 113, 8, 4015, 2634, 0, 0, 20, 19, 48, 2815, 0, 19, 0, 120, 20, 17, 0, 0, 0, 900, 0, 0, 4, 0, 954, 4071, 3367, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7778, 8158, 5520, 5918, 8068, 7294, 8142] dup_rows_per_dst=[292, 0, 2828, 68, 160, 1770, 1348, 0] combine_rows_per_src=[8192, 7778, 8158, 5520, 5918, 8068, 7294, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61466 payload_rows_remote=53292 dup_rows=4070 dup_rows_remote=2266 dedup_ratio=0.062103 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 902, 0, 4087, 0, 0, 3790, 4034, 9, 0, 0, 0, 0, 2431, 0, 0, 0, 0, 0, 0, 14, 80, 0, 283, 0, 0, 19, 0, 955, 4096, 3931, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8082, 8174, 8068, 4862, 7838, 8058, 8192] dup_rows_per_dst=[0, 0, 1804, 18, 0, 534, 1714, 0] combine_rows_per_src=[8192, 8082, 8174, 8068, 4862, 7838, 8058, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63532 payload_rows_remote=55676 dup_rows=2004 dup_rows_remote=1554 dedup_ratio=0.030579 router_tokens_per_expert=[0, 0, 0, 0, 0, 10, 0, 0, 4096, 19, 0, 660, 0, 21, 0, 0, 0, 4031, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 2715, 0, 4096, 783, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7844, 8170, 7856, 8192, 6912, 8174, 8192] dup_rows_per_dst=[20, 1244, 202, 450, 4, 84, 0, 0] combine_rows_per_src=[8192, 7844, 8170, 7856, 8192, 6912, 8174, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=219693056 dispatch_meta_bytes=230160 combine_payload_bytes=219693056 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=219923216 ingress_bytes=270024704 link_bytes=270024704 token_bytes=439616272 weight_movement_bytes=50331648 total_nvlink_bytes=489947920 weight_to_token_ratio=0.114490 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.054576001e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=228499456 dispatch_meta_bytes=229896 combine_payload_bytes=228499456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=228729352 ingress_bytes=253665280 link_bytes=253665280 token_bytes=457228808 weight_movement_bytes=25165824 total_nvlink_bytes=482394632 weight_to_token_ratio=0.055040 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=25216 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.930099687e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=208904192 dispatch_meta_bytes=222792 combine_payload_bytes=208904192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=209126984 ingress_bytes=234070016 link_bytes=234070016 token_bytes=418031176 weight_movement_bytes=25165824 total_nvlink_bytes=443197000 weight_to_token_ratio=0.060201 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781002369e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=218284032 dispatch_meta_bytes=222232 combine_payload_bytes=218284032 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=218506264 ingress_bytes=268615680 link_bytes=268615680 token_bytes=436790296 weight_movement_bytes=50331648 total_nvlink_bytes=487121944 weight_to_token_ratio=0.115231 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.043854957e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=228048896 dispatch_meta_bytes=228920 combine_payload_bytes=228048896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=228277816 ingress_bytes=253214720 link_bytes=253214720 token_bytes=456326712 weight_movement_bytes=25165824 total_nvlink_bytes=481492536 weight_to_token_ratio=0.055149 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=19968 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.926671446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58868 payload_rows_remote=50680 dup_rows=6668 dup_rows_remote=5348 dedup_ratio=0.101746 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 4069, 18, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 538, 0, 4095, 0, 1, 3947, 4027, 0, 0, 1, 1, 1, 2539, 0, 0, 1, 4, 0, 1, 0, 16, 0, 548, 0, 0, 43, 2, 660, 4091, 4093, 1, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 3] payload_rows_per_dst=[8148, 8130, 2514, 8056, 7516, 8128, 8188, 8188] dup_rows_per_dst=[28, 0, 224, 4, 4108, 984, 1320, 0] combine_rows_per_src=[8148, 8130, 2514, 8056, 7516, 8128, 8188, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54454 payload_rows_remote=46306 dup_rows=11082 dup_rows_remote=5262 dedup_ratio=0.169098 router_tokens_per_expert=[0, 3104, 0, 94, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4072, 0, 0, 0, 0, 1112, 0, 0, 0, 0, 99, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 3786, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3560, 0, 2437, 0] payload_rows_per_dst=[8148, 8186, 8192, 7566, 6754, 7388, 8192, 28] dup_rows_per_dst=[5820, 92, 1596, 1976, 2, 1596, 0, 0] combine_rows_per_src=[8148, 8186, 8192, 7566, 6754, 7388, 8192, 28] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56080 payload_rows_remote=47900 dup_rows=9456 dup_rows_remote=9456 dedup_ratio=0.144287 router_tokens_per_expert=[0, 0, 0, 0, 0, 16, 0, 0, 4096, 7, 0, 1441, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3226, 4090, 0, 0, 0, 4060, 0, 0, 0, 3, 0, 0, 3475, 0, 4093, 69, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6958, 2882, 7310, 6524, 8122, 7912, 8192, 8180] dup_rows_per_dst=[26, 14, 2, 1458, 746, 7078, 132, 0] combine_rows_per_src=[6958, 2882, 7310, 6524, 8122, 7912, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=207585280 dispatch_meta_bytes=224112 combine_payload_bytes=207585280 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=207809392 ingress_bytes=358580224 link_bytes=358580224 token_bytes=415394672 weight_movement_bytes=150994944 total_nvlink_bytes=566389616 weight_to_token_ratio=0.363498 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.728381188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57360 payload_rows_remote=49168 dup_rows=8176 dup_rows_remote=2786 dedup_ratio=0.124756 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 4096, 33, 0, 2662, 0, 0, 0, 0, 0, 4058, 0, 183, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 7, 0, 0, 4095, 23, 0, 0, 0, 0, 0, 3985, 0, 4090, 1312, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8190, 34, 8192, 8186, 8192, 8192] dup_rows_per_dst=[4, 5390, 308, 0, 58, 2408, 8, 0] combine_rows_per_src=[8182, 8192, 8190, 34, 8192, 8186, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=189669376 dispatch_meta_bytes=206272 combine_payload_bytes=189669376 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189875648 ingress_bytes=214835200 link_bytes=214835200 token_bytes=379545024 weight_movement_bytes=25165824 total_nvlink_bytes=404710848 weight_to_token_ratio=0.066305 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17152 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.634647644e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=196198400 dispatch_meta_bytes=229424 combine_payload_bytes=196198400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=196427824 ingress_bytes=221364224 link_bytes=221364224 token_bytes=392626224 weight_movement_bytes=25165824 total_nvlink_bytes=417792048 weight_to_token_ratio=0.064096 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.684325972e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58676 payload_rows_remote=52386 dup_rows=6860 dup_rows_remote=6788 dedup_ratio=0.104675 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 0, 4096, 9, 0, 1682, 0, 0, 0, 0, 0, 4095, 0, 1, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 4096, 0, 0, 0, 2989, 192, 0, 0, 0, 0, 0, 3662, 0, 4096, 1890, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3382, 8192, 8192, 6290, 8048, 8188, 8192] dup_rows_per_dst=[16, 0, 78, 3638, 72, 3056, 0, 0] combine_rows_per_src=[8192, 3382, 8192, 8192, 6290, 8048, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54906 payload_rows_remote=51610 dup_rows=10630 dup_rows_remote=10572 dedup_ratio=0.162201 router_tokens_per_expert=[0, 0, 0, 0, 494, 0, 0, 26, 3647, 0, 10, 1356, 0, 97, 0, 0, 0, 4072, 0, 1, 119, 14, 0, 0, 0, 0, 0, 0, 0, 27, 5, 4076, 4072, 0, 0, 5, 2178, 2680, 0, 18, 0, 0, 11, 789, 0, 4071, 857, 20, 0, 38, 0, 0, 0, 0, 4020, 5, 25, 0, 0, 16, 0, 0, 16, 3] payload_rows_per_dst=[8176, 2926, 8166, 8192, 7832, 3296, 8126, 8192] dup_rows_per_dst=[1006, 0, 246, 7318, 1930, 58, 0, 72] combine_rows_per_src=[8176, 2926, 8166, 8192, 7832, 3296, 8126, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64508 payload_rows_remote=56316 dup_rows=1028 dup_rows_remote=1028 dedup_ratio=0.015686 router_tokens_per_expert=[0, 1199, 2614, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 438, 0, 43, 0, 0, 0, 0, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0] payload_rows_per_dst=[7598, 8192, 7802, 8192, 8192, 8192, 8192, 8148] dup_rows_per_dst=[28, 0, 0, 0, 0, 962, 38, 0] combine_rows_per_src=[7598, 8192, 7802, 8192, 8192, 8192, 8192, 8148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59644 payload_rows_remote=51488 dup_rows=5892 dup_rows_remote=5884 dedup_ratio=0.089905 router_tokens_per_expert=[0, 0, 0, 0, 0, 43, 0, 0, 4096, 47, 0, 2685, 0, 0, 0, 0, 0, 4054, 0, 28, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 3084, 4080, 1, 0, 0, 3929, 179, 0, 0, 0, 0, 0, 2331, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8156, 6176, 7890, 4686, 8192, 8160] dup_rows_per_dst=[86, 5464, 8, 0, 328, 6, 0, 0] combine_rows_per_src=[8192, 8192, 8156, 6176, 7890, 4686, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=201392128 dispatch_meta_bytes=207816 combine_payload_bytes=201392128 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=201599944 ingress_bytes=276889600 link_bytes=276889600 token_bytes=402992072 weight_movement_bytes=75497472 total_nvlink_bytes=478489544 weight_to_token_ratio=0.187342 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=16384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.106809928e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=214573056 dispatch_meta_bytes=236696 combine_payload_bytes=214573056 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=214809752 ingress_bytes=264904704 link_bytes=264904704 token_bytes=429382808 weight_movement_bytes=50331648 total_nvlink_bytes=479714456 weight_to_token_ratio=0.117219 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=29184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.015618717e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=211394560 dispatch_meta_bytes=248728 combine_payload_bytes=211394560 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=211643288 ingress_bytes=261726208 link_bytes=261726208 token_bytes=423037848 weight_movement_bytes=50331648 total_nvlink_bytes=473369496 weight_to_token_ratio=0.118977 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.991434035e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=230670336 dispatch_meta_bytes=229376 combine_payload_bytes=230670336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230899712 ingress_bytes=255836160 link_bytes=255836160 token_bytes=461570048 weight_movement_bytes=25165824 total_nvlink_bytes=486735872 weight_to_token_ratio=0.054522 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.946617575e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=210894848 dispatch_meta_bytes=229488 combine_payload_bytes=210894848 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=211124336 ingress_bytes=286392320 link_bytes=286392320 token_bytes=422019184 weight_movement_bytes=75497472 total_nvlink_bytes=497516656 weight_to_token_ratio=0.178896 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.179114647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58192 payload_rows_remote=50052 dup_rows=7344 dup_rows_remote=7344 dedup_ratio=0.112061 router_tokens_per_expert=[0, 0, 0, 0, 0, 53, 0, 0, 4088, 0, 1, 2750, 0, 0, 0, 0, 0, 4095, 0, 0, 53, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3471, 4092, 0, 0, 1, 3869, 90, 0, 1, 1, 0, 1, 1938, 0, 4094, 96, 0, 0, 0, 1, 0, 0, 0, 4069, 0, 1, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8190, 5502, 8192, 8192, 7746, 4042, 8140, 8188] dup_rows_per_dst=[104, 0, 106, 6928, 176, 30, 0, 0] combine_rows_per_src=[8190, 5502, 8192, 8192, 7746, 4042, 8140, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50342 payload_rows_remote=42156 dup_rows=15194 dup_rows_remote=13856 dedup_ratio=0.231842 router_tokens_per_expert=[0, 0, 0, 5, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 0, 0, 4095, 3994, 0, 0, 0, 0, 0, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 0, 0, 0, 524, 4096, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8192, 8190, 5962, 1396, 2032, 8192] dup_rows_per_dst=[1338, 918, 5734, 7146, 0, 0, 58, 0] combine_rows_per_src=[8186, 8192, 8192, 8190, 5962, 1396, 2032, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58568 payload_rows_remote=50376 dup_rows=6968 dup_rows_remote=3106 dedup_ratio=0.106323 router_tokens_per_expert=[0, 2048, 581, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 1529, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[5258, 8192, 8024, 8192, 8188, 8192, 4330, 8192] dup_rows_per_dst=[4, 0, 0, 0, 0, 3058, 44, 3862] combine_rows_per_src=[5258, 8192, 8024, 8192, 8188, 8192, 4330, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6663663120 prefetch_bytes=427819008 reduce_bytes=503316480 weight_movement_bytes=931135488 egress_bytes=3333672464 ingress_bytes=4261126144 link_bytes=4261126144 total_nvlink_bytes=7594798608 weight_to_token_ratio=0.139733 prefetch_share_of_ingress=0.100400 dedup_ratio=0.118645 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=20 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=16510 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.026390406e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57758 payload_rows_remote=49566 dup_rows=7778 dup_rows_remote=7778 dedup_ratio=0.118683 router_tokens_per_expert=[0, 135, 1528, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3740, 3017, 0, 60, 0, 0, 0, 0, 1337, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4096, 0] payload_rows_per_dst=[6380, 8192, 4492, 8172, 8192, 6034, 8192, 8104] dup_rows_per_dst=[1312, 0, 3066, 0, 0, 120, 3280, 0] combine_rows_per_src=[6380, 8192, 4492, 8172, 8192, 6034, 8192, 8104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=205012992 dispatch_meta_bytes=229584 combine_payload_bytes=205012992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=205242576 ingress_bytes=230178816 link_bytes=230178816 token_bytes=410255568 weight_movement_bytes=25165824 total_nvlink_bytes=435421392 weight_to_token_ratio=0.061342 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.751394833e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=172670976 dispatch_meta_bytes=224048 combine_payload_bytes=172670976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=172895024 ingress_bytes=197836800 link_bytes=197836800 token_bytes=345566000 weight_movement_bytes=25165824 total_nvlink_bytes=370731824 weight_to_token_ratio=0.072825 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.505309460e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=206340096 dispatch_meta_bytes=213928 combine_payload_bytes=206340096 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=206554024 ingress_bytes=281837568 link_bytes=281837568 token_bytes=412894120 weight_movement_bytes=75497472 total_nvlink_bytes=488391592 weight_to_token_ratio=0.182849 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.144458247e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65132 payload_rows_remote=56942 dup_rows=404 dup_rows_remote=404 dedup_ratio=0.006165 router_tokens_per_expert=[0, 35, 3929, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 1, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 174, 0, 5, 0, 0, 0, 0, 1, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0] payload_rows_per_dst=[7928, 8192, 8064, 8192, 8190, 8192, 8192, 8182] dup_rows_per_dst=[0, 0, 0, 0, 0, 358, 46, 0] combine_rows_per_src=[7928, 8192, 8064, 8192, 8190, 8192, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64100 payload_rows_remote=55914 dup_rows=1436 dup_rows_remote=262 dedup_ratio=0.021912 router_tokens_per_expert=[0, 181, 3849, 0, 0, 0, 1, 6, 4059, 0, 13, 31, 0, 14, 4012, 0, 0, 0, 0, 0, 35, 8, 4033, 0, 0, 23, 19, 0, 0, 3, 0, 0, 0, 1, 0, 4048, 29, 1, 35, 0, 0, 0, 4065, 574, 22, 0, 0, 19, 4, 0, 41, 0, 28, 6, 22, 0, 0, 61, 0, 0, 0, 3477, 4048, 0] payload_rows_per_dst=[8072, 8126, 8132, 8164, 8170, 8186, 8174, 7076] dup_rows_per_dst=[2, 14, 20, 44, 58, 1174, 124, 0] combine_rows_per_src=[8072, 8126, 8132, 8164, 8170, 8186, 8174, 7076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63070 payload_rows_remote=54890 dup_rows=2466 dup_rows_remote=876 dedup_ratio=0.037628 router_tokens_per_expert=[0, 1934, 1337, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 7, 0, 0, 4090, 913, 0, 6, 0, 0, 0, 0, 53, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 4096, 0] payload_rows_per_dst=[6548, 8192, 8180, 8192, 8192, 7572, 8192, 8002] dup_rows_per_dst=[0, 0, 1590, 0, 10, 672, 194, 0] combine_rows_per_src=[6548, 8192, 8180, 8192, 8192, 7572, 8192, 8002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=203022336 dispatch_meta_bytes=229376 combine_payload_bytes=203022336 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=203251712 ingress_bytes=253353984 link_bytes=253353984 token_bytes=406274048 weight_movement_bytes=50331648 total_nvlink_bytes=456605696 weight_to_token_ratio=0.123886 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.927731084e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5927383232 prefetch_bytes=377487360 reduce_bytes=377487360 weight_movement_bytes=754974720 egress_bytes=2965491904 ingress_bytes=3716866048 link_bytes=3716866048 total_nvlink_bytes=6682357952 weight_to_token_ratio=0.127371 prefetch_share_of_ingress=0.101561 dedup_ratio=0.193741 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=15 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=14859 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.767566001e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=233234432 dispatch_meta_bytes=229384 combine_payload_bytes=233234432 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=233463816 ingress_bytes=283566080 link_bytes=283566080 token_bytes=466698248 weight_movement_bytes=50331648 total_nvlink_bytes=517029896 weight_to_token_ratio=0.107846 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.157610226e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=229023744 dispatch_meta_bytes=224704 combine_payload_bytes=229023744 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=229248448 ingress_bytes=304521216 link_bytes=304521216 token_bytes=458272192 weight_movement_bytes=75497472 total_nvlink_bytes=533769664 weight_to_token_ratio=0.164744 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.317054599e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=224829440 dispatch_meta_bytes=223064 combine_payload_bytes=224829440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=225052504 ingress_bytes=249995264 link_bytes=249995264 token_bytes=449881944 weight_movement_bytes=25165824 total_nvlink_bytes=475047768 weight_to_token_ratio=0.055939 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.902175105e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6054593624 prefetch_bytes=427819008 reduce_bytes=251658240 weight_movement_bytes=679477248 egress_bytes=3029034072 ingress_bytes=3705036800 link_bytes=3705036800 total_nvlink_bytes=6734070872 weight_to_token_ratio=0.112225 prefetch_share_of_ingress=0.115470 dedup_ratio=0.173872 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=10 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=15875 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.761940570e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64066 payload_rows_remote=55876 dup_rows=1470 dup_rows_remote=1314 dedup_ratio=0.022430 router_tokens_per_expert=[2, 434, 3033, 0, 0, 0, 1, 2, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 2, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 4094, 1, 0, 1, 0, 0, 0, 4095, 621, 1, 22, 0, 0, 0, 0, 77, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4063, 4095, 0] payload_rows_per_dst=[6922, 8188, 8060, 8192, 8192, 8192, 8190, 8130] dup_rows_per_dst=[22, 0, 0, 4, 2, 1286, 156, 0] combine_rows_per_src=[6922, 8188, 8060, 8192, 8192, 8192, 8190, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57468 payload_rows_remote=49276 dup_rows=8068 dup_rows_remote=8068 dedup_ratio=0.123108 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3127, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 4096, 0, 0, 0, 3965, 2, 0, 0, 0, 0, 0, 3193, 0, 4096, 1674, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8156, 578, 7952, 8038, 8192, 8168] dup_rows_per_dst=[0, 6312, 54, 0, 6, 1696, 0, 0] combine_rows_per_src=[8192, 8192, 8156, 578, 7952, 8038, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6508323936 prefetch_bytes=377487360 reduce_bytes=377487360 weight_movement_bytes=754974720 egress_bytes=3255993440 ingress_bytes=4007305216 link_bytes=4007305216 total_nvlink_bytes=7263298656 weight_to_token_ratio=0.116001 prefetch_share_of_ingress=0.094200 dedup_ratio=0.129347 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=15 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039292 gemm_inner_pad_upper_bound=16383 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.905685157e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6388054064 prefetch_bytes=427819008 reduce_bytes=352321536 weight_movement_bytes=780140544 egress_bytes=3195828272 ingress_bytes=3972366336 link_bytes=3972366336 total_nvlink_bytes=7168194608 weight_to_token_ratio=0.122125 prefetch_share_of_ingress=0.107699 dedup_ratio=0.147572 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=14 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=16383 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.889069875e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6437320320 prefetch_bytes=251658240 reduce_bytes=528482304 weight_movement_bytes=780140544 egress_bytes=3220477568 ingress_bytes=3996983296 link_bytes=3996983296 total_nvlink_bytes=7217460864 weight_to_token_ratio=0.121190 prefetch_share_of_ingress=0.062962 dedup_ratio=0.133953 prefetch_slots=20 unused_prefetch_slots=236 slot_utilization=0.078125 reduce_slots=21 remote_demand=10 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=15748 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.900776539e-04 prefetch_bytes_per_gemm_flop=1.196767770e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228868096 dispatch_meta_bytes=228760 combine_payload_bytes=228868096 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=229096856 ingress_bytes=254033920 link_bytes=254033920 token_bytes=457964952 weight_movement_bytes=25165824 total_nvlink_bytes=483130776 weight_to_token_ratio=0.054951 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.932904611e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=201834496 dispatch_meta_bytes=229376 combine_payload_bytes=201834496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202063872 ingress_bytes=227000320 link_bytes=227000320 token_bytes=403898368 weight_movement_bytes=25165824 total_nvlink_bytes=429064192 weight_to_token_ratio=0.062307 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.727210151e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6317788864 prefetch_bytes=402653184 reduce_bytes=528482304 weight_movement_bytes=931135488 egress_bytes=3160706752 ingress_bytes=4088217600 link_bytes=4088217600 total_nvlink_bytes=7248924352 weight_to_token_ratio=0.147383 prefetch_share_of_ingress=0.098491 dedup_ratio=0.150816 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=21 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039982 gemm_inner_pad_upper_bound=16891 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.944163266e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57376 payload_rows_remote=49184 dup_rows=8160 dup_rows_remote=890 dedup_ratio=0.124512 router_tokens_per_expert=[0, 2675, 960, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 496, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8106, 8192, 8150, 2512, 8192, 5840] dup_rows_per_dst=[7270, 56, 0, 0, 0, 834, 0, 0] combine_rows_per_src=[8192, 8192, 8106, 8192, 8150, 2512, 8192, 5840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=201457664 dispatch_meta_bytes=200296 combine_payload_bytes=201457664 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=201657960 ingress_bytes=226623488 link_bytes=226623488 token_bytes=403115624 weight_movement_bytes=25165824 total_nvlink_bytes=428281448 weight_to_token_ratio=0.062428 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=17920 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.724342895e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5955227000 prefetch_bytes=377487360 reduce_bytes=150994944 weight_movement_bytes=528482304 egress_bytes=2979368312 ingress_bytes=3504340992 link_bytes=3504340992 total_nvlink_bytes=6483709304 weight_to_token_ratio=0.088743 prefetch_share_of_ingress=0.107720 dedup_ratio=0.193016 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=6 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=15621 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.666499119e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none + [2026-09-21 08:31:46] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 222.6 | throughput per GPU (TFLOP/s/GPU): 144.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.653324E+00 | loss scale: 1.0 | grad norm: 3.499 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45836 payload_rows_remote=39772 dup_rows=19700 dup_rows_remote=16646 dedup_ratio=0.300598 router_tokens_per_expert=[186, 297, 1108, 1, 2, 541, 966, 205, 2, 471, 28, 4, 14, 50, 459, 275, 17, 1170, 812, 1672, 23, 356, 224, 5, 74, 41, 119, 415, 1190, 726, 845, 1555, 787, 255, 423, 1557, 185, 912, 655, 779, 1299, 454, 1138, 178, 719, 855, 15, 31, 804, 851, 627, 83, 347, 1108, 1757, 94, 1550, 24, 272, 7, 36, 27, 1041, 45] payload_rows_per_dst=[6366, 6064, 6622, 4252, 5854, 5376, 5480, 5822] dup_rows_per_dst=[3356, 3054, 3626, 878, 2254, 1726, 2348, 2458] combine_rows_per_src=[6366, 6064, 6622, 4252, 5854, 5376, 5480, 5822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46488 payload_rows_remote=40520 dup_rows=19048 dup_rows_remote=16880 dedup_ratio=0.290649 router_tokens_per_expert=[99, 224, 1111, 0, 0, 593, 717, 404, 14, 388, 24, 3, 3, 44, 389, 142, 7, 1211, 975, 1684, 34, 271, 200, 12, 67, 6, 150, 414, 842, 405, 1278, 1848, 1102, 542, 387, 985, 134, 707, 701, 741, 1382, 562, 1527, 164, 752, 737, 40, 104, 1005, 506, 747, 18, 304, 1671, 1563, 122, 1101, 15, 236, 4, 16, 32, 1258, 44] payload_rows_per_dst=[4860, 5534, 6186, 6586, 5968, 5610, 5818, 5926] dup_rows_per_dst=[1436, 2068, 2602, 3434, 2168, 1872, 2928, 2540] combine_rows_per_src=[4860, 5534, 6186, 6586, 5968, 5610, 5818, 5926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46438 payload_rows_remote=40840 dup_rows=19098 dup_rows_remote=17144 dedup_ratio=0.291412 router_tokens_per_expert=[87, 299, 1504, 0, 0, 449, 696, 437, 0, 459, 117, 1, 9, 11, 769, 126, 2, 770, 1077, 1490, 31, 207, 233, 0, 85, 18, 62, 346, 899, 710, 1198, 1416, 1005, 380, 454, 1005, 231, 1095, 669, 886, 1861, 682, 1533, 89, 726, 358, 8, 52, 939, 955, 402, 35, 110, 1503, 1693, 31, 1212, 13, 366, 2, 5, 26, 900, 34] payload_rows_per_dst=[5202, 5788, 5768, 6304, 6290, 5598, 5610, 5878] dup_rows_per_dst=[1742, 2694, 1852, 3164, 3048, 1954, 2340, 2304] combine_rows_per_src=[5202, 5788, 5768, 6304, 6290, 5598, 5610, 5878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46082 payload_rows_remote=40354 dup_rows=19454 dup_rows_remote=17534 dedup_ratio=0.296844 router_tokens_per_expert=[136, 169, 1055, 2, 0, 637, 1151, 340, 0, 461, 16, 3, 21, 10, 294, 195, 2, 893, 762, 1301, 76, 169, 307, 13, 72, 42, 293, 216, 1024, 867, 1164, 1503, 791, 672, 521, 1313, 232, 1569, 533, 694, 1539, 479, 1679, 70, 660, 1058, 24, 52, 908, 654, 700, 41, 194, 1019, 1959, 68, 1215, 20, 316, 3, 35, 32, 495, 29] payload_rows_per_dst=[5140, 5636, 5324, 6630, 5204, 5780, 6640, 5728] dup_rows_per_dst=[1840, 2128, 1722, 3732, 1682, 1984, 4446, 1920] combine_rows_per_src=[5140, 5636, 5324, 6630, 5204, 5780, 6640, 5728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46152 payload_rows_remote=39862 dup_rows=19384 dup_rows_remote=16662 dedup_ratio=0.295776 router_tokens_per_expert=[278, 469, 1198, 3, 8, 952, 780, 633, 1, 753, 22, 0, 11, 16, 666, 252, 6, 743, 804, 1273, 58, 257, 202, 10, 159, 17, 144, 177, 1562, 580, 1160, 974, 566, 572, 650, 962, 259, 959, 1055, 600, 1550, 368, 1722, 34, 737, 963, 49, 58, 835, 797, 813, 36, 432, 903, 1282, 95, 1148, 54, 294, 3, 19, 55, 681, 49] payload_rows_per_dst=[6308, 5698, 6290, 4710, 6120, 5628, 5574, 5824] dup_rows_per_dst=[3794, 2718, 2722, 1070, 2716, 1890, 2248, 2226] combine_rows_per_src=[6308, 5698, 6290, 4710, 6120, 5628, 5574, 5824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44294 payload_rows_remote=37722 dup_rows=21242 dup_rows_remote=17030 dedup_ratio=0.324127 router_tokens_per_expert=[67, 171, 1448, 4, 11, 652, 713, 643, 2, 552, 29, 5, 3, 10, 446, 120, 7, 928, 959, 1276, 30, 163, 291, 1, 97, 19, 242, 304, 785, 373, 1017, 1683, 1186, 555, 626, 1188, 122, 916, 372, 592, 2136, 394, 1940, 59, 809, 1223, 9, 112, 1070, 542, 611, 72, 134, 1173, 1691, 33, 1105, 12, 264, 7, 25, 13, 690, 36] payload_rows_per_dst=[6572, 5954, 6458, 3168, 5886, 7172, 4402, 4682] dup_rows_per_dst=[4212, 2960, 2886, 486, 2030, 6190, 1030, 1448] combine_rows_per_src=[6572, 5954, 6458, 3168, 5886, 7172, 4402, 4682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46094 payload_rows_remote=39036 dup_rows=19442 dup_rows_remote=15106 dedup_ratio=0.296661 router_tokens_per_expert=[150, 241, 1376, 1, 2, 531, 707, 471, 1, 507, 33, 0, 30, 18, 509, 96, 8, 750, 711, 1251, 20, 234, 305, 0, 58, 13, 113, 399, 1049, 758, 1243, 2064, 925, 619, 716, 937, 211, 1146, 426, 826, 1195, 710, 1852, 202, 846, 603, 18, 108, 958, 556, 624, 60, 183, 1023, 1233, 24, 1440, 18, 336, 11, 9, 23, 1269, 12] payload_rows_per_dst=[5194, 5750, 5232, 7058, 5754, 5568, 4912, 6626] dup_rows_per_dst=[1764, 2654, 1326, 4336, 2308, 1796, 1944, 3314] combine_rows_per_src=[5194, 5750, 5232, 7058, 5754, 5568, 4912, 6626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 3, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162906112 dispatch_meta_bytes=225672 combine_payload_bytes=162906112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=163131784 ingress_bytes=238403584 link_bytes=238403584 token_bytes=326037896 weight_movement_bytes=75497472 total_nvlink_bytes=401535368 weight_to_token_ratio=0.231560 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.036765 gemm_active_groups=12 gemm_peak_group_rows=9088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.813975814e-04 prefetch_bytes_per_gemm_flop=5.744485294e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165969920 dispatch_meta_bytes=229600 combine_payload_bytes=165969920 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=166199520 ingress_bytes=266633216 link_bytes=266633216 token_bytes=332169440 weight_movement_bytes=100663296 total_nvlink_bytes=432832736 weight_to_token_ratio=0.303048 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=9856 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.028770696e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=167280640 dispatch_meta_bytes=231936 combine_payload_bytes=167280640 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=167512576 ingress_bytes=217612288 link_bytes=217612288 token_bytes=334793216 weight_movement_bytes=50331648 total_nvlink_bytes=385124864 weight_to_token_ratio=0.150337 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.655778075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165289984 dispatch_meta_bytes=231552 combine_payload_bytes=165289984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=165521536 ingress_bytes=190455808 link_bytes=190455808 token_bytes=330811520 weight_movement_bytes=25165824 total_nvlink_bytes=355977344 weight_to_token_ratio=0.076073 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.449148639e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163274752 dispatch_meta_bytes=226096 combine_payload_bytes=163274752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=163500848 ingress_bytes=188440576 link_bytes=188440576 token_bytes=326775600 weight_movement_bytes=25165824 total_nvlink_bytes=351941424 weight_to_token_ratio=0.077013 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.433815052e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=154509312 dispatch_meta_bytes=219008 combine_payload_bytes=154509312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=154728320 ingress_bytes=179675136 link_bytes=179675136 token_bytes=309237632 weight_movement_bytes=25165824 total_nvlink_bytes=334403456 weight_to_token_ratio=0.081380 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.367120182e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159891456 dispatch_meta_bytes=216568 combine_payload_bytes=159891456 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160108024 ingress_bytes=235388928 link_bytes=235388928 token_bytes=319999480 weight_movement_bytes=75497472 total_nvlink_bytes=395496952 weight_to_token_ratio=0.235930 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791037765e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46650 payload_rows_remote=40422 dup_rows=18886 dup_rows_remote=15298 dedup_ratio=0.288177 router_tokens_per_expert=[225, 137, 1697, 0, 0, 820, 661, 335, 0, 678, 65, 1, 14, 8, 499, 111, 15, 705, 904, 1352, 58, 103, 323, 0, 81, 30, 100, 209, 779, 898, 983, 1784, 1578, 522, 442, 1097, 179, 945, 229, 643, 2145, 361, 2309, 53, 555, 552, 5, 25, 931, 543, 645, 45, 113, 1218, 1934, 13, 1137, 9, 266, 2, 6, 32, 643, 16] payload_rows_per_dst=[5608, 5184, 5350, 6378, 5772, 5800, 6228, 6330] dup_rows_per_dst=[2142, 1652, 1570, 3350, 2482, 1592, 3588, 2510] combine_rows_per_src=[5608, 5184, 5350, 6378, 5772, 5800, 6228, 6330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48502 payload_rows_remote=41170 dup_rows=17034 dup_rows_remote=11852 dedup_ratio=0.259918 router_tokens_per_expert=[1972, 5, 0, 0, 1, 380, 0, 685, 25, 1905, 1020, 2890, 3, 0, 1984, 0, 12, 0, 44, 189, 253, 22, 95, 1, 3, 2493, 0, 0, 913, 827, 354, 96, 3, 0, 0, 0, 20, 693, 638, 0, 319, 0, 1676, 2, 0, 0, 927, 0, 0, 0, 63, 3436, 3526, 0, 0, 0, 2160, 0, 1680, 1273, 152, 0, 28, 0] payload_rows_per_dst=[6280, 7332, 4874, 4850, 5448, 7766, 7102, 4850] dup_rows_per_dst=[2352, 5182, 632, 1620, 1112, 4954, 76, 1106] combine_rows_per_src=[6280, 7332, 4874, 4850, 5448, 7766, 7102, 4850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51568 payload_rows_remote=46822 dup_rows=13968 dup_rows_remote=13208 dedup_ratio=0.213135 router_tokens_per_expert=[1886, 0, 2, 0, 3, 47, 0, 1640, 2, 1091, 2151, 2204, 0, 0, 678, 16, 0, 0, 112, 27, 156, 92, 336, 0, 0, 3305, 0, 0, 627, 3151, 300, 0, 0, 0, 7, 0, 577, 833, 658, 0, 6, 0, 34, 2, 0, 0, 896, 0, 0, 35, 41, 3925, 3669, 0, 0, 0, 688, 0, 3252, 261, 39, 0, 19, 0] payload_rows_per_dst=[5894, 5422, 5130, 8082, 4746, 7904, 7390, 7000] dup_rows_per_dst=[1262, 1098, 724, 6684, 760, 1822, 100, 1518] combine_rows_per_src=[5894, 5422, 5130, 8082, 4746, 7904, 7390, 7000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49010 payload_rows_remote=41804 dup_rows=16526 dup_rows_remote=14110 dedup_ratio=0.252167 router_tokens_per_expert=[1606, 0, 0, 0, 4, 200, 0, 2516, 1, 348, 1204, 2428, 0, 0, 2948, 0, 0, 0, 3, 183, 882, 194, 45, 0, 0, 734, 0, 4, 51, 1248, 976, 0, 0, 1, 0, 0, 297, 201, 552, 0, 274, 0, 257, 0, 0, 0, 909, 3, 0, 0, 609, 3958, 3281, 0, 0, 0, 2137, 0, 1383, 3195, 4, 0, 132, 0] payload_rows_per_dst=[6578, 2848, 6246, 4750, 6620, 7206, 7172, 7590] dup_rows_per_dst=[2074, 258, 1224, 1276, 1378, 2416, 1788, 6112] combine_rows_per_src=[6578, 2848, 6246, 4750, 6620, 7206, 7172, 7590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45794 payload_rows_remote=38988 dup_rows=19742 dup_rows_remote=16238 dedup_ratio=0.301239 router_tokens_per_expert=[1038, 5, 38, 0, 0, 271, 0, 410, 23, 1876, 2832, 3040, 3, 2, 1628, 1, 0, 0, 470, 13, 2729, 368, 63, 0, 0, 1357, 0, 0, 62, 534, 1467, 0, 2, 0, 0, 0, 1554, 630, 244, 0, 356, 0, 376, 4, 1, 0, 276, 0, 32, 2, 345, 3709, 2388, 2, 8, 5, 2617, 1, 847, 612, 11, 3, 509, 4] payload_rows_per_dst=[3342, 6786, 7844, 5270, 5998, 1910, 7838, 6806] dup_rows_per_dst=[182, 2688, 5522, 1570, 2118, 116, 4042, 3504] combine_rows_per_src=[3342, 6786, 7844, 5270, 5998, 1910, 7838, 6806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45494 payload_rows_remote=37348 dup_rows=20042 dup_rows_remote=12050 dedup_ratio=0.305817 router_tokens_per_expert=[2766, 0, 0, 0, 0, 747, 5, 581, 18, 640, 953, 3176, 3, 0, 1067, 11, 1, 0, 47, 0, 401, 1694, 333, 39, 1, 153, 0, 0, 107, 1210, 1915, 117, 1, 0, 0, 2, 405, 1506, 558, 7, 1492, 0, 1025, 0, 0, 0, 236, 0, 2, 0, 114, 3412, 2504, 0, 22, 1, 366, 0, 1012, 3970, 64, 0, 84, 0] payload_rows_per_dst=[8146, 7680, 4284, 5788, 4464, 7756, 5120, 2256] dup_rows_per_dst=[7992, 4362, 746, 1644, 494, 4574, 166, 64] combine_rows_per_src=[8146, 7680, 4284, 5788, 4464, 7756, 5120, 2256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47888 payload_rows_remote=39880 dup_rows=17648 dup_rows_remote=13972 dedup_ratio=0.269287 router_tokens_per_expert=[107, 4, 2, 0, 0, 450, 0, 459, 0, 1658, 1587, 3682, 1, 0, 2032, 0, 12, 1, 27, 511, 1414, 66, 129, 0, 0, 2869, 0, 0, 27, 2814, 417, 0, 0, 0, 10, 0, 1041, 1265, 766, 0, 31, 0, 582, 5, 0, 15, 309, 0, 4, 0, 96, 2420, 2145, 27, 14, 0, 2426, 0, 1680, 311, 411, 0, 941, 0] payload_rows_per_dst=[2486, 6716, 8008, 6320, 6860, 6036, 4464, 6998] dup_rows_per_dst=[180, 4004, 3676, 1706, 3368, 688, 108, 3918] combine_rows_per_src=[2486, 6716, 8008, 6320, 6860, 6036, 4464, 6998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=165568512 dispatch_meta_bytes=222880 combine_payload_bytes=165568512 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=165791392 ingress_bytes=241065984 link_bytes=241065984 token_bytes=331359904 weight_movement_bytes=75497472 total_nvlink_bytes=406857376 weight_to_token_ratio=0.227841 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8704 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.834233602e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51878 payload_rows_remote=46380 dup_rows=13658 dup_rows_remote=11830 dedup_ratio=0.208405 router_tokens_per_expert=[2210, 0, 0, 0, 11, 1215, 1, 1664, 9, 2570, 1005, 3766, 0, 0, 912, 0, 8, 0, 375, 24, 50, 38, 336, 4, 0, 930, 0, 0, 473, 1686, 573, 1, 0, 2, 1, 0, 891, 1442, 564, 0, 195, 0, 593, 0, 0, 0, 1307, 0, 1, 0, 55, 1713, 3657, 2, 0, 0, 1040, 0, 157, 2797, 30, 0, 460, 0] payload_rows_per_dst=[7258, 6032, 7768, 5498, 5700, 6170, 7336, 6116] dup_rows_per_dst=[3696, 1136, 1434, 1828, 1924, 1446, 94, 2100] combine_rows_per_src=[7258, 6032, 7768, 5498, 5700, 6170, 7336, 6116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=168632320 dispatch_meta_bytes=212088 combine_payload_bytes=168632320 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=168844408 ingress_bytes=244129792 link_bytes=244129792 token_bytes=337476728 weight_movement_bytes=75497472 total_nvlink_bytes=412974200 weight_to_token_ratio=0.223712 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.857545641e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=191782912 dispatch_meta_bytes=240120 combine_payload_bytes=191782912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=192023032 ingress_bytes=216948736 link_bytes=216948736 token_bytes=383805944 weight_movement_bytes=25165824 total_nvlink_bytes=408971768 weight_to_token_ratio=0.065569 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.650729211e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=171229184 dispatch_meta_bytes=223656 combine_payload_bytes=171229184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=171452840 ingress_bytes=196395008 link_bytes=196395008 token_bytes=342682024 weight_movement_bytes=25165824 total_nvlink_bytes=367847848 weight_to_token_ratio=0.073438 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.494339089e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=220904 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159915752 ingress_bytes=235192320 link_bytes=235192320 token_bytes=319610600 weight_movement_bytes=75497472 total_nvlink_bytes=395108072 weight_to_token_ratio=0.236217 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789541805e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=152977408 dispatch_meta_bytes=197592 combine_payload_bytes=152977408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=153175000 ingress_bytes=178143232 link_bytes=178143232 token_bytes=306152408 weight_movement_bytes=25165824 total_nvlink_bytes=331318232 weight_to_token_ratio=0.082200 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.355464162e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=163348480 dispatch_meta_bytes=215408 combine_payload_bytes=163348480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=163563888 ingress_bytes=188514304 link_bytes=188514304 token_bytes=326912368 weight_movement_bytes=25165824 total_nvlink_bytes=352078192 weight_to_token_ratio=0.076980 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.434376037e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=189972480 dispatch_meta_bytes=232840 combine_payload_bytes=189972480 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=190205320 ingress_bytes=240304128 link_bytes=240304128 token_bytes=380177800 weight_movement_bytes=50331648 total_nvlink_bytes=430509448 weight_to_token_ratio=0.132390 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.828436758e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46384 payload_rows_remote=38674 dup_rows=19152 dup_rows_remote=14122 dedup_ratio=0.292236 router_tokens_per_expert=[772, 0, 0, 0, 2, 1608, 4, 1521, 64, 213, 1719, 1383, 0, 0, 2778, 0, 0, 0, 464, 43, 1477, 104, 241, 0, 0, 1447, 0, 0, 966, 1181, 727, 4, 0, 0, 0, 0, 349, 586, 3560, 0, 1526, 1, 17, 0, 0, 0, 842, 0, 6, 2, 498, 2805, 3054, 0, 4, 1, 127, 0, 180, 2205, 258, 0, 29, 0] payload_rows_per_dst=[6062, 3832, 5448, 5738, 8142, 4268, 7710, 5184] dup_rows_per_dst=[1752, 160, 1976, 2912, 6404, 504, 5030, 414] combine_rows_per_src=[6062, 3832, 5448, 5738, 8142, 4268, 7710, 5184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48614 payload_rows_remote=42234 dup_rows=16922 dup_rows_remote=15468 dedup_ratio=0.258209 router_tokens_per_expert=[0, 43, 2, 1, 436, 0, 115, 0, 1558, 0, 17, 193, 0, 2126, 0, 23, 544, 0, 4044, 41, 32, 1, 0, 0, 0, 0, 0, 1566, 0, 28, 0, 0, 304, 1206, 3, 6, 98, 4039, 2147, 0, 0, 0, 7, 0, 4026, 1, 0, 0, 30, 21, 4082, 3688, 0, 1, 35, 0, 0, 417, 7, 1, 858, 798, 221, 2] payload_rows_per_dst=[8160, 6380, 8192, 8184, 5872, 8068, 174, 3584] dup_rows_per_dst=[1112, 1454, 8508, 3168, 1656, 0, 0, 1024] combine_rows_per_src=[8160, 6380, 8192, 8184, 5872, 8068, 174, 3584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52868 payload_rows_remote=45764 dup_rows=12668 dup_rows_remote=10648 dedup_ratio=0.193298 router_tokens_per_expert=[2, 0, 0, 0, 776, 0, 624, 15, 663, 0, 0, 107, 0, 3238, 0, 14, 1405, 9, 3913, 1, 22, 0, 0, 0, 1, 0, 0, 652, 0, 0, 1, 0, 8, 1407, 4, 0, 4, 2601, 3139, 0, 0, 0, 1, 0, 2212, 7, 0, 0, 2, 77, 4096, 3384, 0, 71, 192, 0, 0, 541, 6, 0, 1043, 1157, 1373, 0] payload_rows_per_dst=[6290, 6856, 4270, 8192, 7104, 7186, 6998, 5972] dup_rows_per_dst=[1746, 1188, 310, 1308, 2020, 3374, 454, 2268] combine_rows_per_src=[6290, 6856, 4270, 8192, 7104, 7186, 6998, 5972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51090 payload_rows_remote=44436 dup_rows=14446 dup_rows_remote=13016 dedup_ratio=0.220428 router_tokens_per_expert=[0, 2, 0, 0, 2194, 0, 691, 0, 2258, 0, 0, 33, 0, 50, 0, 22, 2085, 0, 2703, 0, 1, 0, 0, 0, 0, 0, 0, 518, 10, 0, 0, 0, 0, 2133, 2176, 0, 226, 2225, 1037, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 50, 9, 4082, 1537, 0, 129, 2474, 0, 1, 1923, 14, 0, 143, 738, 1965, 0] payload_rows_per_dst=[6798, 4636, 4172, 8168, 7318, 6654, 6534, 6810] dup_rows_per_dst=[3426, 90, 0, 1052, 3826, 1430, 1864, 2758] combine_rows_per_src=[6798, 4636, 4172, 8168, 7318, 6654, 6534, 6810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=41544 payload_rows_remote=35552 dup_rows=23992 dup_rows_remote=20596 dedup_ratio=0.366089 router_tokens_per_expert=[0, 18, 0, 15, 175, 0, 135, 1, 1849, 0, 0, 91, 2, 2033, 0, 1340, 835, 2, 3990, 0, 0, 0, 0, 3, 0, 3, 0, 1391, 1, 0, 0, 0, 1, 2499, 5, 6, 24, 1319, 1639, 0, 4, 0, 0, 83, 2301, 0, 0, 0, 1158, 0, 4055, 3174, 0, 58, 951, 0, 2, 2060, 0, 0, 163, 743, 638, 1] payload_rows_per_dst=[2514, 6836, 1680, 2790, 5404, 8146, 8182, 5992] dup_rows_per_dst=[210, 3794, 0, 0, 688, 4610, 11294, 3396] combine_rows_per_src=[2514, 6836, 1680, 2790, 5404, 8146, 8182, 5992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52216 payload_rows_remote=44080 dup_rows=13320 dup_rows_remote=9210 dedup_ratio=0.203247 router_tokens_per_expert=[0, 0, 123, 0, 56, 0, 306, 0, 3888, 4, 0, 34, 0, 1300, 0, 229, 1207, 148, 3960, 0, 0, 0, 0, 0, 0, 2, 0, 1055, 0, 261, 0, 0, 0, 1220, 81, 0, 43, 4032, 2925, 5, 0, 0, 0, 0, 2532, 12, 1, 0, 1088, 0, 4080, 1761, 0, 28, 407, 1, 0, 79, 0, 0, 31, 1785, 84, 0] payload_rows_per_dst=[8078, 7876, 8136, 8192, 6894, 5090, 4092, 3858] dup_rows_per_dst=[956, 3034, 4110, 2604, 1654, 0, 862, 100] combine_rows_per_src=[8078, 7876, 8136, 8192, 6894, 5090, 4092, 3858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49650 payload_rows_remote=45658 dup_rows=15886 dup_rows_remote=15786 dedup_ratio=0.242401 router_tokens_per_expert=[0, 23, 1, 0, 66, 2, 19, 0, 3860, 4, 1, 248, 16, 350, 0, 43, 1339, 52, 3648, 12, 1, 8, 0, 0, 0, 5, 0, 896, 0, 0, 0, 2, 58, 1033, 0, 0, 47, 1935, 3726, 0, 0, 0, 0, 1, 2573, 0, 2, 0, 174, 0, 3963, 1938, 1, 653, 1535, 14, 0, 3643, 2, 0, 167, 338, 369, 0] payload_rows_per_dst=[3992, 8026, 7982, 7988, 5122, 5152, 3898, 7490] dup_rows_per_dst=[100, 4166, 6014, 1744, 1458, 0, 856, 1548] combine_rows_per_src=[3992, 8026, 7982, 7988, 5122, 5152, 3898, 7490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53068 payload_rows_remote=44876 dup_rows=12468 dup_rows_remote=11550 dedup_ratio=0.190247 router_tokens_per_expert=[0, 22, 8, 0, 886, 0, 349, 0, 2797, 0, 0, 22, 0, 807, 0, 2, 587, 3, 3694, 9, 0, 0, 0, 0, 0, 0, 0, 453, 4, 4, 0, 0, 1, 4047, 105, 5, 28, 2996, 2227, 0, 0, 0, 0, 5, 2254, 0, 0, 0, 184, 1, 4094, 2175, 0, 68, 703, 0, 0, 1917, 4, 0, 182, 799, 1324, 2] payload_rows_per_dst=[6800, 6318, 7506, 8192, 8168, 4518, 5214, 6352] dup_rows_per_dst=[1722, 938, 1080, 918, 4658, 0, 1048, 2104] combine_rows_per_src=[6800, 6318, 7506, 8192, 8168, 4518, 5214, 6352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158408704 dispatch_meta_bytes=211184 combine_payload_bytes=158408704 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158619888 ingress_bytes=208740352 link_bytes=208740352 token_bytes=317028592 weight_movement_bytes=50331648 total_nvlink_bytes=367360240 weight_to_token_ratio=0.158761 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588272893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=172990464 dispatch_meta_bytes=230808 combine_payload_bytes=172990464 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=173221272 ingress_bytes=198156288 link_bytes=198156288 token_bytes=346211736 weight_movement_bytes=25165824 total_nvlink_bytes=371377560 weight_to_token_ratio=0.072689 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.507740395e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=187449344 dispatch_meta_bytes=225648 combine_payload_bytes=187449344 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=187674992 ingress_bytes=288112640 link_bytes=288112640 token_bytes=375124336 weight_movement_bytes=100663296 total_nvlink_bytes=475787632 weight_to_token_ratio=0.268346 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.192204295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=182009856 dispatch_meta_bytes=229808 combine_payload_bytes=182009856 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=182239664 ingress_bytes=207175680 link_bytes=207175680 token_bytes=364249520 weight_movement_bytes=25165824 total_nvlink_bytes=389415344 weight_to_token_ratio=0.069090 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.576367547e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=145620992 dispatch_meta_bytes=224592 combine_payload_bytes=145620992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=145845584 ingress_bytes=170786816 link_bytes=170786816 token_bytes=291466576 weight_movement_bytes=25165824 total_nvlink_bytes=316632400 weight_to_token_ratio=0.086342 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.299490336e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=180551680 dispatch_meta_bytes=213160 combine_payload_bytes=180551680 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=180764840 ingress_bytes=230883328 link_bytes=230883328 token_bytes=361316520 weight_movement_bytes=50331648 total_nvlink_bytes=411648168 weight_to_token_ratio=0.139301 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.756755355e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=187015168 dispatch_meta_bytes=245776 combine_payload_bytes=187015168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=187260944 ingress_bytes=212180992 link_bytes=212180992 token_bytes=374276112 weight_movement_bytes=25165824 total_nvlink_bytes=399441936 weight_to_token_ratio=0.067239 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.614452188e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=183812096 dispatch_meta_bytes=225704 combine_payload_bytes=183812096 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=184037800 ingress_bytes=208977920 link_bytes=208977920 token_bytes=367849896 weight_movement_bytes=25165824 total_nvlink_bytes=393015720 weight_to_token_ratio=0.068413 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590080511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44238 payload_rows_remote=36754 dup_rows=21298 dup_rows_remote=14716 dedup_ratio=0.324982 router_tokens_per_expert=[0, 1, 0, 0, 1244, 0, 1857, 0, 1050, 0, 1, 805, 0, 91, 0, 25, 1788, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 255, 16, 8, 0, 0, 7, 979, 34, 0, 104, 3714, 723, 0, 0, 0, 0, 0, 1854, 0, 1, 0, 1066, 64, 4079, 1664, 0, 551, 848, 0, 0, 2224, 12, 0, 21, 1996, 1887, 0] payload_rows_per_dst=[7940, 3744, 3576, 2996, 3240, 7932, 7484, 7326] dup_rows_per_dst=[5692, 200, 0, 40, 454, 3376, 6582, 4954] combine_rows_per_src=[7940, 3744, 3576, 2996, 3240, 7932, 7484, 7326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51938 payload_rows_remote=45020 dup_rows=13598 dup_rows_remote=12238 dedup_ratio=0.207489 router_tokens_per_expert=[0, 0, 1, 1, 3, 51, 8, 0, 0, 991, 0, 0, 0, 3137, 2, 9, 0, 0, 3611, 2409, 0, 0, 1, 0, 0, 0, 0, 6, 37, 3870, 0, 4096, 0, 0, 2412, 0, 46, 7, 0, 0, 11, 1, 3637, 62, 0, 1, 0, 42, 0, 3230, 0, 0, 781, 0, 236, 0, 0, 0, 15, 0, 4054, 0, 0, 0] payload_rows_per_dst=[6512, 6918, 7842, 8192, 4886, 7422, 2028, 8138] dup_rows_per_dst=[76, 1360, 4200, 7826, 44, 86, 6, 0] combine_rows_per_src=[6512, 6918, 7842, 8192, 4886, 7422, 2028, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56762 payload_rows_remote=48572 dup_rows=8774 dup_rows_remote=6444 dedup_ratio=0.133881 router_tokens_per_expert=[0, 0, 0, 0, 0, 394, 0, 0, 0, 2758, 13, 0, 0, 3100, 0, 97, 2, 0, 3520, 820, 0, 0, 9, 0, 0, 0, 0, 188, 1, 919, 0, 4089, 0, 2, 1167, 0, 2, 0, 0, 0, 0, 0, 3858, 258, 0, 0, 0, 19, 0, 3552, 0, 0, 4020, 0, 27, 0, 0, 0, 1, 1, 3951, 0, 0, 0] payload_rows_per_dst=[7396, 7694, 7362, 2156, 8190, 8014, 8046, 7904] dup_rows_per_dst=[496, 4242, 1340, 60, 2330, 256, 48, 2] combine_rows_per_src=[7396, 7694, 7362, 2156, 8190, 8014, 8046, 7904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51020 payload_rows_remote=43954 dup_rows=14516 dup_rows_remote=14334 dedup_ratio=0.221497 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 2, 0, 0, 768, 31, 0, 0, 3819, 0, 605, 0, 0, 3736, 876, 5, 0, 0, 0, 0, 0, 0, 84, 0, 647, 0, 4096, 4, 0, 2856, 0, 0, 0, 0, 0, 0, 0, 3529, 25, 0, 0, 0, 2, 0, 3824, 0, 0, 3960, 0, 11, 0, 0, 0, 0, 0, 3865, 21, 0, 0] payload_rows_per_dst=[7652, 7866, 1974, 2390, 8192, 7066, 8150, 7730] dup_rows_per_dst=[4, 2580, 2, 100, 5720, 182, 5886, 42] combine_rows_per_src=[7652, 7866, 1974, 2390, 8192, 7066, 8150, 7730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47416 payload_rows_remote=40206 dup_rows=18120 dup_rows_remote=14984 dedup_ratio=0.276489 router_tokens_per_expert=[0, 3, 0, 0, 0, 1204, 0, 2, 0, 566, 419, 0, 0, 187, 0, 547, 0, 23, 3671, 3222, 0, 0, 79, 7, 0, 5, 4, 85, 9, 1304, 4, 4051, 0, 7, 3727, 0, 1, 20, 6, 0, 0, 0, 2891, 0, 10, 0, 0, 0, 0, 3866, 0, 0, 3380, 10, 390, 0, 1, 0, 0, 2, 3063, 0, 2, 0] payload_rows_per_dst=[7846, 3064, 6606, 200, 8192, 6698, 7600, 7210] dup_rows_per_dst=[2304, 302, 128, 14, 7432, 1712, 3092, 3136] combine_rows_per_src=[7846, 3064, 6606, 200, 8192, 6698, 7600, 7210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48544 payload_rows_remote=40630 dup_rows=16992 dup_rows_remote=11850 dedup_ratio=0.259277 router_tokens_per_expert=[0, 2, 0, 0, 0, 37, 0, 5, 0, 197, 1840, 2, 0, 2597, 4, 740, 0, 31, 3484, 2873, 0, 0, 140, 0, 0, 2, 0, 6, 4, 3454, 3, 4090, 0, 8, 543, 0, 0, 0, 3, 0, 0, 6, 437, 0, 0, 0, 0, 262, 0, 3301, 0, 10, 3215, 0, 1389, 0, 2, 1, 6, 0, 4071, 3, 0, 0] payload_rows_per_dst=[6652, 7304, 7914, 8190, 1104, 1408, 7812, 8160] dup_rows_per_dst=[38, 3456, 5142, 6928, 4, 2, 1416, 6] combine_rows_per_src=[6652, 7304, 7914, 8190, 1104, 1408, 7812, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56540 payload_rows_remote=48824 dup_rows=8996 dup_rows_remote=3212 dedup_ratio=0.137268 router_tokens_per_expert=[11, 0, 0, 0, 0, 3, 0, 10, 0, 164, 213, 0, 0, 2688, 0, 0, 0, 23, 1293, 1853, 0, 0, 41, 0, 0, 0, 0, 30, 1, 3734, 0, 4061, 0, 0, 1975, 0, 0, 0, 0, 0, 0, 0, 4061, 91, 0, 0, 0, 31, 0, 4056, 0, 0, 3992, 0, 431, 0, 0, 0, 0, 0, 4005, 0, 0, 1] payload_rows_per_dst=[8126, 5826, 5592, 7716, 5108, 8166, 7994, 8012] dup_rows_per_dst=[34, 304, 828, 5784, 994, 200, 852, 0] combine_rows_per_src=[8126, 5826, 5592, 7716, 5108, 8166, 7994, 8012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47178 payload_rows_remote=39036 dup_rows=18358 dup_rows_remote=11026 dedup_ratio=0.280121 router_tokens_per_expert=[16, 4, 0, 0, 0, 791, 0, 0, 0, 2033, 14, 2, 0, 613, 1, 427, 5, 0, 3902, 1111, 2, 36, 0, 0, 0, 0, 0, 44, 37, 3771, 2, 4089, 1, 0, 789, 1, 0, 0, 0, 2, 6, 1, 4071, 262, 2, 0, 0, 3, 0, 3178, 0, 5, 3907, 5, 0, 0, 7, 5, 0, 0, 3589, 0, 4, 30] payload_rows_per_dst=[8142, 5540, 7966, 8188, 1586, 8172, 338, 7246] dup_rows_per_dst=[7332, 640, 2146, 7698, 0, 518, 0, 24] combine_rows_per_src=[8142, 5540, 7966, 8188, 1586, 8172, 338, 7246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=150544384 dispatch_meta_bytes=205880 combine_payload_bytes=150544384 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=150750264 ingress_bytes=226041856 link_bytes=226041856 token_bytes=301294648 weight_movement_bytes=75497472 total_nvlink_bytes=376792120 weight_to_token_ratio=0.250577 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.719917347e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=184401920 dispatch_meta_bytes=229032 combine_payload_bytes=184401920 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=184630952 ingress_bytes=209567744 link_bytes=209567744 token_bytes=369032872 weight_movement_bytes=25165824 total_nvlink_bytes=394198696 weight_to_token_ratio=0.068194 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=20224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594568390e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=198950912 dispatch_meta_bytes=220064 combine_payload_bytes=198950912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199170976 ingress_bytes=224116736 link_bytes=224116736 token_bytes=398121888 weight_movement_bytes=25165824 total_nvlink_bytes=423287712 weight_to_token_ratio=0.063211 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.705269408e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=180035584 dispatch_meta_bytes=233152 combine_payload_bytes=180035584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=180268736 ingress_bytes=205201408 link_bytes=205201408 token_bytes=360304320 weight_movement_bytes=25165824 total_nvlink_bytes=385470144 weight_to_token_ratio=0.069846 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.561345618e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=164683776 dispatch_meta_bytes=220760 combine_payload_bytes=164683776 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=164904536 ingress_bytes=189849600 link_bytes=189849600 token_bytes=329588312 weight_movement_bytes=25165824 total_nvlink_bytes=354754136 weight_to_token_ratio=0.076355 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.444536097e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=166420480 dispatch_meta_bytes=209920 combine_payload_bytes=166420480 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=166630400 ingress_bytes=267083776 link_bytes=267083776 token_bytes=333050880 weight_movement_bytes=100663296 total_nvlink_bytes=433714176 weight_to_token_ratio=0.302246 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.032198937e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=199983104 dispatch_meta_bytes=208144 combine_payload_bytes=199983104 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=200191248 ingress_bytes=275480576 link_bytes=275480576 token_bytes=400174352 weight_movement_bytes=75497472 total_nvlink_bytes=475671824 weight_to_token_ratio=0.188661 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.096088883e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159891456 dispatch_meta_bytes=200248 combine_payload_bytes=159891456 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=160091704 ingress_bytes=210223104 link_bytes=210223104 token_bytes=319983160 weight_movement_bytes=50331648 total_nvlink_bytes=370314808 weight_to_token_ratio=0.157295 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=26496 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599554922e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49892 payload_rows_remote=41922 dup_rows=15644 dup_rows_remote=11346 dedup_ratio=0.238708 router_tokens_per_expert=[18, 0, 0, 0, 0, 18, 0, 0, 0, 11, 4, 0, 0, 4042, 0, 163, 0, 0, 3213, 3246, 0, 1, 0, 0, 0, 0, 1, 46, 0, 3713, 0, 4087, 0, 0, 1790, 0, 0, 0, 0, 0, 0, 0, 3542, 2, 0, 0, 0, 560, 0, 1427, 0, 0, 2010, 0, 911, 0, 0, 0, 0, 0, 3958, 1, 0, 4] payload_rows_per_dst=[2914, 8148, 6494, 92, 8176, 8182, 7970, 7916] dup_rows_per_dst=[12, 292, 0, 2, 3578, 7452, 4298, 10] combine_rows_per_src=[2914, 8148, 6494, 92, 8176, 8182, 7970, 7916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57890 payload_rows_remote=49702 dup_rows=7646 dup_rows_remote=7646 dedup_ratio=0.116669 router_tokens_per_expert=[2, 3634, 4, 0, 1838, 0, 1, 74, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 840, 0, 0, 0, 0, 407, 14, 0, 321, 0, 0, 0, 0, 0, 0, 5, 0, 2, 3985, 2380, 3218, 4, 3360, 422, 0, 1, 0, 25, 0, 267, 0, 28, 0, 3772, 0, 11, 0, 0, 0, 1, 4054] payload_rows_per_dst=[7984, 8188, 8154, 5386, 8004, 5748, 6872, 7554] dup_rows_per_dst=[3122, 0, 1642, 858, 1784, 20, 206, 14] combine_rows_per_src=[7984, 8188, 8154, 5386, 8004, 5748, 6872, 7554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57364 payload_rows_remote=49200 dup_rows=8172 dup_rows_remote=8168 dedup_ratio=0.124695 router_tokens_per_expert=[0, 3988, 0, 0, 3696, 0, 0, 2, 4069, 0, 3, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 295, 0, 0, 0, 0, 177, 0, 0, 29, 0, 0, 4, 0, 0, 0, 0, 0, 1, 4080, 902, 4069, 5, 2669, 728, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 6, 0, 0, 0, 0, 4027] payload_rows_per_dst=[8184, 8144, 8060, 2212, 8164, 6426, 8140, 8034] dup_rows_per_dst=[7188, 0, 594, 4, 4, 380, 0, 2] combine_rows_per_src=[8184, 8144, 8060, 2212, 8164, 6426, 8140, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59298 payload_rows_remote=52128 dup_rows=6238 dup_rows_remote=2690 dedup_ratio=0.095184 router_tokens_per_expert=[0, 3760, 0, 0, 885, 3, 0, 23, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 2286, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3640, 1359, 4066, 1, 2919, 1343, 0, 0, 0, 0, 0, 104, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 3955] payload_rows_per_dst=[6054, 8192, 8100, 6188, 7288, 7170, 8144, 8162] dup_rows_per_dst=[1096, 2, 294, 1102, 0, 3548, 196, 0] combine_rows_per_src=[6054, 8192, 8100, 6188, 7288, 7170, 8144, 8162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51704 payload_rows_remote=45434 dup_rows=13832 dup_rows_remote=8832 dedup_ratio=0.211060 router_tokens_per_expert=[0, 3776, 0, 15, 1433, 0, 0, 273, 4062, 0, 0, 0, 143, 1, 5, 0, 34, 0, 0, 0, 8, 20, 0, 175, 2, 5, 2, 0, 122, 0, 0, 1976, 0, 1, 0, 0, 0, 0, 22, 0, 0, 3938, 2028, 4069, 0, 3011, 411, 0, 0, 1, 0, 0, 134, 13, 15, 15, 2776, 2, 182, 0, 7, 0, 0, 4091] payload_rows_per_dst=[3230, 8100, 3174, 6886, 7920, 7936, 8188, 6270] dup_rows_per_dst=[212, 272, 196, 1384, 2, 5474, 1292, 5000] combine_rows_per_src=[3230, 8100, 3174, 6886, 7920, 7936, 8188, 6270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62596 payload_rows_remote=54444 dup_rows=2940 dup_rows_remote=2926 dedup_ratio=0.044861 router_tokens_per_expert=[0, 3726, 0, 0, 881, 0, 1, 0, 4083, 37, 0, 0, 50, 1, 10, 0, 0, 0, 0, 3, 8, 0, 1, 4, 14, 0, 0, 0, 0, 0, 0, 544, 0, 3, 0, 3, 10, 0, 60, 0, 0, 4086, 3521, 4007, 0, 3624, 192, 0, 0, 0, 0, 1, 5, 0, 255, 6, 3518, 7, 34, 0, 6, 0, 0, 4067] payload_rows_per_dst=[8118, 8192, 8152, 7464, 8192, 7264, 8122, 7092] dup_rows_per_dst=[1098, 170, 14, 694, 132, 368, 426, 38] combine_rows_per_src=[8118, 8192, 8152, 7464, 8192, 7264, 8122, 7092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62008 payload_rows_remote=54372 dup_rows=3528 dup_rows_remote=3344 dedup_ratio=0.053833 router_tokens_per_expert=[0, 3675, 0, 0, 566, 0, 0, 5, 4096, 0, 0, 0, 69, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 604, 0, 0, 0, 2, 0, 0, 0, 104, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3995, 3804, 4082, 3, 1818, 2517, 0, 0, 0, 0, 0, 24, 5, 11, 0, 3625, 0, 0, 0, 0, 0, 0, 3755] payload_rows_per_dst=[7616, 8192, 7738, 7636, 7998, 7414, 8164, 7250] dup_rows_per_dst=[876, 146, 980, 184, 0, 1262, 80, 0] combine_rows_per_src=[7616, 8192, 7738, 7636, 7998, 7414, 8164, 7250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 2, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=203579392 dispatch_meta_bytes=229392 combine_payload_bytes=203579392 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=203808784 ingress_bytes=253911040 link_bytes=253911040 token_bytes=407388176 weight_movement_bytes=50331648 total_nvlink_bytes=457719824 weight_to_token_ratio=0.123547 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.931969636e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=171712512 dispatch_meta_bytes=213072 combine_payload_bytes=171712512 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=171925584 ingress_bytes=247209984 link_bytes=247209984 token_bytes=343638096 weight_movement_bytes=75497472 total_nvlink_bytes=419135568 weight_to_token_ratio=0.219701 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=21632 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.880982343e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56540 payload_rows_remote=49994 dup_rows=8996 dup_rows_remote=7406 dedup_ratio=0.137268 router_tokens_per_expert=[0, 1094, 0, 29, 2422, 0, 1, 329, 4064, 1, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 1, 0, 0, 928, 0, 0, 2, 0, 18, 0, 6, 22, 1, 7, 0, 0, 3, 0, 0, 0, 17, 4079, 4033, 3867, 0, 2714, 867, 0, 0, 0, 1, 80, 6, 0, 0, 4, 3911, 9, 134, 1, 4, 0, 0, 4088] payload_rows_per_dst=[6546, 7744, 8190, 8172, 8172, 1768, 7898, 8050] dup_rows_per_dst=[1590, 0, 1894, 5234, 192, 0, 18, 68] combine_rows_per_src=[6546, 7744, 8190, 8172, 8172, 1768, 7898, 8050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=201523200 dispatch_meta_bytes=229472 combine_payload_bytes=201523200 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=201752672 ingress_bytes=251854848 link_bytes=251854848 token_bytes=403275872 weight_movement_bytes=50331648 total_nvlink_bytes=453607520 weight_to_token_ratio=0.124807 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.916324391e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=213516288 dispatch_meta_bytes=219272 combine_payload_bytes=213516288 prefetch_bytes=25165824 reduce_bytes=150994944 egress_bytes=213735560 ingress_bytes=389677056 link_bytes=389677056 token_bytes=427251848 weight_movement_bytes=176160768 total_nvlink_bytes=603412616 weight_to_token_ratio=0.412311 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17280 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.964992149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=186097664 dispatch_meta_bytes=217064 combine_payload_bytes=186097664 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=186314728 ingress_bytes=236429312 link_bytes=236429312 token_bytes=372412392 weight_movement_bytes=50331648 total_nvlink_bytes=422744040 weight_to_token_ratio=0.135150 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.798953885e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=223002624 dispatch_meta_bytes=229480 combine_payload_bytes=223002624 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=223232104 ingress_bytes=248168448 link_bytes=248168448 token_bytes=446234728 weight_movement_bytes=25165824 total_nvlink_bytes=471400552 weight_to_token_ratio=0.056396 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.888275146e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222707712 dispatch_meta_bytes=230864 combine_payload_bytes=222707712 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=222938576 ingress_bytes=273039360 link_bytes=273039360 token_bytes=445646288 weight_movement_bytes=50331648 total_nvlink_bytes=495977936 weight_to_token_ratio=0.112941 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.077514050e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=204775424 dispatch_meta_bytes=229600 combine_payload_bytes=204775424 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=205005024 ingress_bytes=255107072 link_bytes=255107072 token_bytes=409780448 weight_movement_bytes=50331648 total_nvlink_bytes=460112096 weight_to_token_ratio=0.122826 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.941070058e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49444 payload_rows_remote=41436 dup_rows=16092 dup_rows_remote=14492 dedup_ratio=0.245544 router_tokens_per_expert=[7, 0, 27, 1070, 0, 2063, 13, 0, 10, 0, 0, 3985, 0, 809, 0, 0, 0, 1898, 3633, 7, 0, 0, 0, 0, 1933, 290, 917, 0, 0, 618, 0, 0, 619, 0, 6, 0, 19, 13, 561, 11, 0, 0, 0, 0, 1234, 4082, 0, 2183, 1, 0, 80, 0, 0, 0, 0, 0, 2963, 0, 4, 3711, 0, 0, 0, 1] payload_rows_per_dst=[7360, 8008, 7932, 7798, 2286, 6096, 4028, 5936] dup_rows_per_dst=[5926, 1600, 4382, 3274, 172, 738, 0, 0] combine_rows_per_src=[7360, 8008, 7932, 7798, 2286, 6096, 4028, 5936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58312 payload_rows_remote=50620 dup_rows=7224 dup_rows_remote=7126 dedup_ratio=0.110229 router_tokens_per_expert=[0, 3644, 0, 0, 1223, 2, 0, 86, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 4, 0, 0, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 3868, 3827, 0, 1211, 524, 0, 0, 0, 0, 0, 8, 0, 60, 0, 4001, 0, 0, 0, 0, 0, 0, 4082] payload_rows_per_dst=[2582, 8192, 8164, 8112, 7684, 7884, 7692, 8002] dup_rows_per_dst=[40, 2, 190, 4020, 0, 2874, 98, 0] combine_rows_per_src=[2582, 8192, 8164, 8112, 7684, 7884, 7692, 8002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51604 payload_rows_remote=44182 dup_rows=13932 dup_rows_remote=13292 dedup_ratio=0.212585 router_tokens_per_expert=[0, 0, 2, 1232, 0, 5, 0, 0, 0, 0, 0, 3128, 0, 1631, 0, 0, 0, 3385, 1674, 44, 0, 1, 0, 0, 3774, 3699, 36, 0, 0, 3923, 0, 0, 848, 0, 0, 0, 0, 25, 30, 0, 0, 0, 0, 5, 1, 4019, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 2319, 7, 0, 2785, 0, 0, 0, 0] payload_rows_per_dst=[2476, 3262, 7506, 7818, 7422, 8038, 7600, 7482] dup_rows_per_dst=[2, 0, 2702, 2694, 640, 402, 4752, 2740] combine_rows_per_src=[2476, 3262, 7506, 7818, 7422, 8038, 7600, 7482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50348 payload_rows_remote=43940 dup_rows=15188 dup_rows_remote=13448 dedup_ratio=0.231750 router_tokens_per_expert=[0, 0, 0, 3749, 0, 67, 0, 0, 0, 0, 0, 3930, 0, 3261, 0, 0, 0, 2264, 218, 4, 0, 0, 0, 0, 3751, 525, 55, 0, 0, 3913, 0, 0, 1501, 0, 0, 0, 0, 9, 37, 0, 0, 0, 0, 0, 37, 2867, 0, 1170, 0, 0, 185, 0, 0, 0, 0, 0, 3466, 0, 0, 1759, 0, 0, 0, 0] payload_rows_per_dst=[7532, 6522, 4858, 1154, 8008, 6408, 8188, 7678] dup_rows_per_dst=[100, 0, 114, 6, 2946, 1740, 7510, 2772] combine_rows_per_src=[7532, 6522, 4858, 1154, 8008, 6408, 8188, 7678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50704 payload_rows_remote=42910 dup_rows=14832 dup_rows_remote=14234 dedup_ratio=0.226318 router_tokens_per_expert=[0, 0, 0, 1301, 0, 2313, 0, 0, 0, 0, 0, 4088, 0, 3253, 0, 1, 6, 1074, 1287, 0, 4, 0, 96, 3, 1733, 1800, 75, 0, 0, 4085, 0, 0, 178, 0, 0, 0, 36, 0, 364, 0, 0, 0, 0, 0, 268, 3280, 5, 3275, 0, 2, 0, 8, 8, 0, 21, 8, 322, 0, 0, 3859, 0, 3, 0, 12] payload_rows_per_dst=[6430, 4434, 4066, 3728, 8190, 7870, 8192, 7794] dup_rows_per_dst=[798, 0, 874, 22, 1142, 7860, 3538, 598] combine_rows_per_src=[6430, 4434, 4066, 3728, 8190, 7870, 8192, 7794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=43998 payload_rows_remote=36170 dup_rows=21538 dup_rows_remote=12674 dedup_ratio=0.328644 router_tokens_per_expert=[0, 0, 0, 1448, 0, 22, 0, 0, 0, 3, 0, 4070, 0, 2155, 4, 0, 9, 2228, 2961, 0, 1, 0, 198, 0, 1030, 2389, 163, 1, 6, 1087, 5, 0, 3643, 3, 10, 9, 0, 58, 376, 0, 0, 0, 0, 0, 553, 3796, 7, 1166, 0, 0, 450, 0, 0, 0, 14, 10, 1081, 0, 0, 3803, 0, 7, 2, 0] payload_rows_per_dst=[2934, 7990, 7828, 8170, 7442, 4496, 2958, 2180] dup_rows_per_dst=[6, 3984, 8864, 6738, 1246, 650, 50, 0] combine_rows_per_src=[2934, 7990, 7828, 8170, 7442, 4496, 2958, 2180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=169721856 dispatch_meta_bytes=223712 combine_payload_bytes=169721856 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=169945568 ingress_bytes=245219328 link_bytes=245219328 token_bytes=339667424 weight_movement_bytes=75497472 total_nvlink_bytes=415164896 weight_to_token_ratio=0.222269 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=20992 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.865835751e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55356 payload_rows_remote=47388 dup_rows=10180 dup_rows_remote=4292 dedup_ratio=0.155334 router_tokens_per_expert=[0, 0, 0, 2394, 0, 1519, 0, 0, 0, 0, 0, 4080, 0, 3952, 2, 0, 0, 3212, 543, 12, 0, 0, 0, 0, 968, 590, 163, 0, 0, 3430, 0, 0, 782, 0, 0, 0, 0, 14, 32, 0, 0, 1, 0, 0, 106, 3685, 0, 115, 0, 0, 161, 0, 0, 0, 0, 0, 3916, 5, 0, 3086, 0, 0, 0, 0] payload_rows_per_dst=[6638, 7904, 7188, 7968, 8166, 7382, 2254, 7856] dup_rows_per_dst=[1188, 4, 346, 5888, 1650, 432, 4, 668] combine_rows_per_src=[6638, 7904, 7188, 7968, 8166, 7382, 2254, 7856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=207339520 dispatch_meta_bytes=230984 combine_payload_bytes=207339520 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=207570504 ingress_bytes=257671168 link_bytes=257671168 token_bytes=414910024 weight_movement_bytes=50331648 total_nvlink_bytes=465241672 weight_to_token_ratio=0.121307 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.960579866e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46524 payload_rows_remote=38340 dup_rows=19012 dup_rows_remote=11622 dedup_ratio=0.290100 router_tokens_per_expert=[0, 17, 0, 3516, 0, 3, 170, 0, 0, 2, 0, 4008, 0, 3916, 2, 0, 4, 419, 401, 0, 5, 0, 0, 0, 4053, 1876, 250, 1, 9, 2031, 5, 0, 998, 1, 0, 0, 0, 61, 3150, 0, 0, 0, 0, 0, 0, 4081, 13, 226, 0, 0, 0, 3, 5, 0, 0, 10, 3006, 0, 0, 518, 0, 8, 0, 0] payload_rows_per_dst=[8184, 8146, 1498, 7194, 6920, 478, 8116, 5988] dup_rows_per_dst=[7390, 7750, 160, 2186, 1500, 0, 26, 0] combine_rows_per_src=[8184, 8146, 1498, 7194, 6920, 478, 8116, 5988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=180969472 dispatch_meta_bytes=229896 combine_payload_bytes=180969472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=181199368 ingress_bytes=206135296 link_bytes=206135296 token_bytes=362168840 weight_movement_bytes=25165824 total_nvlink_bytes=387334664 weight_to_token_ratio=0.069486 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.568451426e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=179978240 dispatch_meta_bytes=229552 combine_payload_bytes=179978240 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=180207792 ingress_bytes=255475712 link_bytes=255475712 token_bytes=360186032 weight_movement_bytes=75497472 total_nvlink_bytes=435683504 weight_to_token_ratio=0.209607 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.943874982e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=175759360 dispatch_meta_bytes=228576 combine_payload_bytes=175759360 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=175987936 ingress_bytes=226091008 link_bytes=226091008 token_bytes=351747296 weight_movement_bytes=50331648 total_nvlink_bytes=402078944 weight_to_token_ratio=0.143090 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=19968 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.720291337e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=148152320 dispatch_meta_bytes=195376 combine_payload_bytes=148152320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=148347696 ingress_bytes=173318144 link_bytes=173318144 token_bytes=296500016 weight_movement_bytes=25165824 total_nvlink_bytes=321665840 weight_to_token_ratio=0.084876 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.318750818e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=194101248 dispatch_meta_bytes=206720 combine_payload_bytes=194101248 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=194307968 ingress_bytes=269598720 link_bytes=269598720 token_bytes=388409216 weight_movement_bytes=75497472 total_nvlink_bytes=463906688 weight_to_token_ratio=0.194376 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=11264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.051334755e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=157040640 dispatch_meta_bytes=199848 combine_payload_bytes=157040640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=157240488 ingress_bytes=182206464 link_bytes=182206464 token_bytes=314281128 weight_movement_bytes=25165824 total_nvlink_bytes=339446952 weight_to_token_ratio=0.080074 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.386380663e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55750 payload_rows_remote=51302 dup_rows=9786 dup_rows_remote=8930 dedup_ratio=0.149323 router_tokens_per_expert=[72, 0, 38, 0, 1, 164, 0, 0, 0, 1, 0, 6, 1367, 1277, 0, 1, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 2145, 4091, 0, 4081, 0, 2983, 11, 0, 0, 0, 4075, 2, 96, 2, 0, 1, 393, 0, 0, 0, 2026, 2, 0, 0, 6, 0, 0, 2161, 3721, 0, 2, 0, 0, 0, 0, 4031, 0, 0] payload_rows_per_dst=[8192, 4448, 8172, 6208, 8178, 8152, 4334, 8066] dup_rows_per_dst=[540, 856, 1630, 2432, 194, 4134, 0, 0] combine_rows_per_src=[8192, 4448, 8172, 6208, 8178, 8152, 4334, 8066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52676 payload_rows_remote=44856 dup_rows=12860 dup_rows_remote=12360 dedup_ratio=0.196228 router_tokens_per_expert=[0, 0, 0, 4034, 0, 252, 0, 0, 0, 0, 0, 3537, 0, 2966, 0, 0, 0, 3173, 370, 66, 0, 0, 0, 0, 3752, 21, 428, 0, 0, 331, 0, 0, 126, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 7, 3067, 0, 3399, 0, 0, 77, 0, 0, 0, 0, 0, 2849, 0, 0, 4073, 0, 0, 0, 0] payload_rows_per_dst=[8102, 5932, 6480, 898, 7212, 8042, 7820, 8190] dup_rows_per_dst=[470, 0, 738, 0, 594, 4904, 500, 5654] combine_rows_per_src=[8102, 5932, 6480, 898, 7212, 8042, 7820, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=210132992 dispatch_meta_bytes=240928 combine_payload_bytes=210132992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210373920 ingress_bytes=235298816 link_bytes=235298816 token_bytes=420506912 weight_movement_bytes=25165824 total_nvlink_bytes=445672736 weight_to_token_ratio=0.059846 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790352117e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56646 payload_rows_remote=48462 dup_rows=8890 dup_rows_remote=7844 dedup_ratio=0.135651 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 3507, 0, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4095, 1, 4008, 0, 3272, 8, 0, 0, 0, 4084, 0, 523, 0, 0, 1, 91, 0, 0, 0, 398, 11, 0, 0, 1, 1, 0, 712, 4032, 0, 0, 0, 0, 0, 0, 3735, 0, 0] payload_rows_per_dst=[8192, 7056, 8018, 8174, 8184, 8124, 1428, 7470] dup_rows_per_dst=[0, 428, 0, 6474, 1046, 942, 0, 0] combine_rows_per_src=[8192, 7056, 8018, 8174, 8184, 8124, 1428, 7470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57712 payload_rows_remote=49524 dup_rows=7824 dup_rows_remote=5682 dedup_ratio=0.119385 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 571, 3247, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2157, 4094, 0, 3850, 0, 2090, 0, 0, 0, 0, 4096, 0, 3411, 0, 0, 0, 1002, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 580, 4074, 0, 0, 0, 0, 0, 0, 3503, 0, 0] payload_rows_per_dst=[8190, 6906, 7702, 6616, 7002, 8188, 6102, 7006] dup_rows_per_dst=[0, 732, 2, 1878, 2168, 2142, 902, 0] combine_rows_per_src=[8190, 6906, 7702, 6616, 7002, 8188, 6102, 7006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49082 payload_rows_remote=40916 dup_rows=16454 dup_rows_remote=11990 dedup_ratio=0.251068 router_tokens_per_expert=[56, 0, 0, 0, 65, 0, 0, 7, 16, 0, 0, 0, 109, 1953, 0, 28, 0, 0, 0, 0, 0, 3, 34, 1, 4, 0, 2264, 4043, 95, 3326, 0, 3919, 0, 0, 0, 0, 4058, 0, 2214, 0, 0, 0, 956, 7, 0, 1, 568, 0, 0, 14, 14, 0, 0, 890, 4067, 5, 0, 4, 5, 0, 0, 4018, 13, 11] payload_rows_per_dst=[6728, 4154, 6722, 7998, 4428, 2698, 8188, 8166] dup_rows_per_dst=[156, 58, 6, 1496, 0, 366, 9908, 4464] combine_rows_per_src=[6728, 4154, 6722, 7998, 4428, 2698, 8188, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56790 payload_rows_remote=48912 dup_rows=8746 dup_rows_remote=5532 dedup_ratio=0.133453 router_tokens_per_expert=[192, 0, 0, 0, 8, 0, 0, 4, 12, 0, 0, 2, 1138, 3473, 0, 88, 0, 4, 0, 0, 0, 257, 12, 1600, 0, 0, 1291, 4075, 3, 3673, 0, 857, 98, 0, 12, 0, 4068, 3, 1041, 0, 0, 0, 465, 12, 0, 9, 9, 0, 0, 0, 0, 0, 0, 2165, 4088, 2, 1, 0, 7, 0, 0, 4086, 13, 0] payload_rows_per_dst=[8174, 7800, 7878, 4062, 8168, 8192, 4334, 8182] dup_rows_per_dst=[384, 1626, 3214, 240, 2276, 974, 0, 32] combine_rows_per_src=[8174, 7800, 7878, 4062, 8168, 8192, 4334, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58944 payload_rows_remote=51412 dup_rows=6592 dup_rows_remote=4532 dedup_ratio=0.100586 router_tokens_per_expert=[19, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 614, 3832, 0, 0, 2, 0, 0, 0, 0, 7, 2, 274, 0, 0, 3266, 4080, 0, 4092, 0, 1530, 7, 0, 0, 0, 4096, 0, 871, 0, 0, 0, 346, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 1844, 3903, 0, 0, 0, 0, 0, 0, 3682, 0, 0] payload_rows_per_dst=[8174, 7924, 8186, 7532, 8192, 7884, 3688, 7364] dup_rows_per_dst=[30, 968, 568, 2060, 1756, 1210, 0, 0] combine_rows_per_src=[8174, 7924, 8186, 7532, 8192, 7884, 3688, 7364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55774 payload_rows_remote=47598 dup_rows=9762 dup_rows_remote=9512 dedup_ratio=0.148956 router_tokens_per_expert=[16, 0, 3, 0, 3, 64, 60, 0, 11, 6, 0, 0, 0, 3696, 0, 78, 0, 11, 0, 20, 1, 0, 0, 0, 0, 10, 1323, 4067, 19, 4035, 0, 2949, 6, 0, 0, 0, 4059, 0, 878, 0, 0, 0, 961, 16, 0, 3, 37, 281, 0, 0, 0, 21, 38, 3415, 4009, 0, 1, 0, 5, 0, 0, 2651, 15, 0] payload_rows_per_dst=[8176, 7886, 8132, 3532, 7744, 8074, 6908, 5322] dup_rows_per_dst=[250, 5410, 186, 20, 1294, 2540, 40, 22] combine_rows_per_src=[8176, 7886, 8132, 3532, 7744, 8074, 6908, 5322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=183730176 dispatch_meta_bytes=228864 combine_payload_bytes=183730176 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=183959040 ingress_bytes=234061824 link_bytes=234061824 token_bytes=367689216 weight_movement_bytes=50331648 total_nvlink_bytes=418020864 weight_to_token_ratio=0.136886 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.780940037e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=202850304 dispatch_meta_bytes=220824 combine_payload_bytes=202850304 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203071128 ingress_bytes=228016128 link_bytes=228016128 token_bytes=405921432 weight_movement_bytes=25165824 total_nvlink_bytes=431087256 weight_to_token_ratio=0.061997 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=24192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.734939276e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=167591936 dispatch_meta_bytes=211624 combine_payload_bytes=167591936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=167803560 ingress_bytes=192757760 link_bytes=192757760 token_bytes=335395496 weight_movement_bytes=25165824 total_nvlink_bytes=360561320 weight_to_token_ratio=0.075033 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.466663834e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=198500352 dispatch_meta_bytes=225224 combine_payload_bytes=198500352 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=198725576 ingress_bytes=248832000 link_bytes=248832000 token_bytes=397225928 weight_movement_bytes=50331648 total_nvlink_bytes=447557576 weight_to_token_ratio=0.126708 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=21248 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.893324011e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=200343552 dispatch_meta_bytes=217776 combine_payload_bytes=200343552 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=200561328 ingress_bytes=250675200 link_bytes=250675200 token_bytes=400904880 weight_movement_bytes=50331648 total_nvlink_bytes=451236528 weight_to_token_ratio=0.125545 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.907348633e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54720 payload_rows_remote=46562 dup_rows=10816 dup_rows_remote=10816 dedup_ratio=0.165039 router_tokens_per_expert=[0, 0, 0, 64, 1, 0, 2034, 30, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 289, 0, 37, 4019, 0, 0, 0, 0, 177, 0, 17, 0, 2, 0, 0, 2602, 4004, 3950, 36, 0, 0, 0, 2267, 0, 5, 0, 648, 3224, 0, 9, 0, 0, 0, 2, 56, 0, 0, 0, 3958, 0, 0, 0, 1258, 0] payload_rows_per_dst=[8046, 8158, 8192, 8060, 8050, 5140, 6558, 2516] dup_rows_per_dst=[4128, 0, 498, 4704, 762, 700, 24, 0] combine_rows_per_src=[8046, 8158, 8192, 8060, 8050, 5140, 6558, 2516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=210583552 dispatch_meta_bytes=223776 combine_payload_bytes=210583552 prefetch_bytes=25165824 reduce_bytes=125829120 egress_bytes=210807328 ingress_bytes=361578496 link_bytes=361578496 token_bytes=421390880 weight_movement_bytes=150994944 total_nvlink_bytes=572385824 weight_to_token_ratio=0.358325 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.751194574e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=194961408 dispatch_meta_bytes=228440 combine_payload_bytes=194961408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=195189848 ingress_bytes=220127232 link_bytes=220127232 token_bytes=390151256 weight_movement_bytes=25165824 total_nvlink_bytes=415317080 weight_to_token_ratio=0.064503 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.674913893e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=190717952 dispatch_meta_bytes=229512 combine_payload_bytes=190717952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=190947464 ingress_bytes=215883776 link_bytes=215883776 token_bytes=381665416 weight_movement_bytes=25165824 total_nvlink_bytes=406831240 weight_to_token_ratio=0.065937 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.642626095e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59592 payload_rows_remote=52006 dup_rows=5944 dup_rows_remote=3140 dedup_ratio=0.090698 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 2412, 0, 0, 0, 7, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 3905, 3501, 3827, 7, 0, 0, 0, 625, 0, 0, 0, 1069, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 0, 0, 0, 2504, 2] payload_rows_per_dst=[4830, 7732, 8190, 7820, 7668, 7586, 7980, 7786] dup_rows_per_dst=[0, 14, 4, 16, 0, 2804, 0, 3106] combine_rows_per_src=[4830, 7732, 8190, 7820, 7668, 7586, 7980, 7786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57578 payload_rows_remote=51884 dup_rows=7958 dup_rows_remote=7234 dedup_ratio=0.121429 router_tokens_per_expert=[0, 0, 0, 34, 5, 0, 3067, 69, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 41, 33, 0, 0, 0, 0, 0, 0, 3088, 3366, 2764, 2, 0, 0, 2, 152, 0, 1, 0, 2212, 3738, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 0, 0, 0, 2334, 134] payload_rows_per_dst=[6294, 8088, 8180, 6278, 5694, 7628, 7518, 7898] dup_rows_per_dst=[56, 0, 0, 46, 724, 2952, 48, 4132] combine_rows_per_src=[6294, 8088, 8180, 6278, 5694, 7628, 7518, 7898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52880 payload_rows_remote=45930 dup_rows=12656 dup_rows_remote=11196 dedup_ratio=0.193115 router_tokens_per_expert=[5, 3, 16, 1186, 0, 0, 3786, 0, 0, 11, 0, 0, 0, 0, 2504, 122, 0, 0, 8, 0, 0, 1, 27, 4085, 0, 2, 0, 0, 0, 2, 0, 11, 0, 0, 16, 3966, 1223, 1492, 0, 0, 24, 0, 3614, 3, 57, 0, 3084, 3291, 0, 0, 0, 4, 0, 0, 0, 20, 0, 2, 2927, 0, 0, 0, 552, 724] payload_rows_per_dst=[7676, 5232, 5978, 7954, 3926, 7728, 7436, 6950] dup_rows_per_dst=[2316, 42, 16, 8, 1338, 3118, 4358, 1460] combine_rows_per_src=[7676, 5232, 5978, 7954, 3926, 7728, 7436, 6950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56942 payload_rows_remote=48764 dup_rows=8594 dup_rows_remote=8022 dedup_ratio=0.131134 router_tokens_per_expert=[3, 7, 8, 15, 0, 0, 3012, 0, 0, 8, 216, 0, 0, 4, 4001, 50, 0, 0, 9, 0, 0, 0, 36, 4086, 14, 1, 0, 0, 0, 5, 220, 2, 0, 1, 22, 3400, 3157, 1094, 0, 0, 11, 0, 1761, 13, 5, 3, 1683, 4048, 0, 0, 0, 3, 0, 0, 5, 0, 4, 0, 4059, 1, 0, 0, 1797, 4] payload_rows_per_dst=[7946, 8084, 8178, 7144, 7152, 6404, 8102, 3932] dup_rows_per_dst=[5002, 474, 572, 140, 1396, 548, 10, 452] combine_rows_per_src=[7946, 8084, 8178, 7144, 7152, 6404, 8102, 3932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49038 payload_rows_remote=40846 dup_rows=16498 dup_rows_remote=4986 dedup_ratio=0.251740 router_tokens_per_expert=[0, 0, 9, 0, 1, 69, 0, 0, 0, 0, 0, 0, 549, 4093, 0, 0, 0, 0, 0, 0, 0, 6, 0, 97, 0, 0, 3203, 4096, 0, 2457, 0, 1546, 0, 0, 0, 0, 4096, 0, 2453, 0, 0, 0, 552, 0, 0, 0, 108, 0, 0, 0, 1, 0, 0, 1968, 4085, 0, 0, 0, 0, 0, 0, 3379, 0, 0] payload_rows_per_dst=[8192, 8186, 5056, 5878, 4906, 1734, 8192, 6894] dup_rows_per_dst=[158, 1098, 64, 936, 0, 182, 11512, 2548] combine_rows_per_src=[8192, 8186, 5056, 5878, 4906, 1734, 8192, 6894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57068 payload_rows_remote=49754 dup_rows=8468 dup_rows_remote=8468 dedup_ratio=0.129211 router_tokens_per_expert=[0, 0, 0, 244, 0, 0, 1785, 9, 0, 0, 139, 0, 0, 0, 3453, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 0, 0, 0, 8, 9, 0, 0, 0, 0, 0, 0, 3640, 4010, 2883, 0, 0, 0, 0, 816, 0, 0, 0, 2788, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 957, 77] payload_rows_per_dst=[4016, 7048, 8190, 7314, 8172, 6464, 8026, 7838] dup_rows_per_dst=[60, 136, 8, 0, 5614, 744, 0, 1906] combine_rows_per_src=[4016, 7048, 8190, 7314, 8172, 6464, 8026, 7838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53486 payload_rows_remote=45712 dup_rows=12050 dup_rows_remote=6876 dedup_ratio=0.183868 router_tokens_per_expert=[1, 18, 0, 0, 0, 0, 3609, 0, 0, 10, 104, 0, 2, 43, 3612, 23, 0, 11, 0, 11, 18, 8, 514, 4079, 0, 1, 0, 0, 0, 5, 70, 0, 0, 0, 9, 3485, 3521, 3000, 0, 0, 17, 0, 40, 4, 0, 12, 2091, 3177, 0, 476, 0, 12, 0, 0, 0, 0, 0, 0, 2846, 2, 0, 0, 1629, 308] payload_rows_per_dst=[7774, 8114, 8188, 7360, 7060, 4310, 6878, 3802] dup_rows_per_dst=[5174, 4614, 1094, 622, 0, 18, 452, 76] combine_rows_per_src=[7774, 8114, 8188, 7360, 7060, 4310, 6878, 3802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=213016576 dispatch_meta_bytes=220584 combine_payload_bytes=213016576 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=213237160 ingress_bytes=288514048 link_bytes=288514048 token_bytes=426253736 weight_movement_bytes=75497472 total_nvlink_bytes=501751208 weight_to_token_ratio=0.177119 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.195258546e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=212516864 dispatch_meta_bytes=236472 combine_payload_bytes=212516864 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=212753336 ingress_bytes=338345984 link_bytes=338345984 token_bytes=425270200 weight_movement_bytes=125829120 total_nvlink_bytes=551099320 weight_to_token_ratio=0.295880 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.574422001e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61104 payload_rows_remote=53252 dup_rows=4432 dup_rows_remote=4020 dedup_ratio=0.067627 router_tokens_per_expert=[2, 3956, 0, 470, 0, 0, 4053, 1, 4043, 205, 0, 0, 3926, 0, 0, 1, 16, 0, 0, 4091, 0, 0, 0, 4087, 0, 0, 11, 7, 53, 0, 0, 0, 0, 4, 0, 4078, 754, 0, 0, 0, 0, 0, 0, 2291, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7918, 7852, 8174, 8118, 8162, 4582, 8192, 8106] dup_rows_per_dst=[940, 412, 32, 110, 1510, 0, 1428, 0] combine_rows_per_src=[7918, 7852, 8174, 8118, 8162, 4582, 8192, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=188129280 dispatch_meta_bytes=228504 combine_payload_bytes=188129280 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=188357784 ingress_bytes=238460928 link_bytes=238460928 token_bytes=376487064 weight_movement_bytes=50331648 total_nvlink_bytes=426818712 weight_to_token_ratio=0.133688 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.814412136e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=199737344 dispatch_meta_bytes=227144 combine_payload_bytes=199737344 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=199964488 ingress_bytes=250068992 link_bytes=250068992 token_bytes=399701832 weight_movement_bytes=50331648 total_nvlink_bytes=450033480 weight_to_token_ratio=0.125923 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.902736090e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=167305216 dispatch_meta_bytes=183328 combine_payload_bytes=167305216 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=167488544 ingress_bytes=217636864 link_bytes=217636864 token_bytes=334793760 weight_movement_bytes=50331648 total_nvlink_bytes=385125408 weight_to_token_ratio=0.150336 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=13824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.655965070e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=203792384 dispatch_meta_bytes=232888 combine_payload_bytes=203792384 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=204025272 ingress_bytes=254124032 link_bytes=254124032 token_bytes=407817656 weight_movement_bytes=50331648 total_nvlink_bytes=458149304 weight_to_token_ratio=0.123417 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.933590259e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=187236352 dispatch_meta_bytes=210352 combine_payload_bytes=187236352 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=187446704 ingress_bytes=212402176 link_bytes=212402176 token_bytes=374683056 weight_movement_bytes=25165824 total_nvlink_bytes=399848880 weight_to_token_ratio=0.067166 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.616135142e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=218120192 dispatch_meta_bytes=229088 combine_payload_bytes=218120192 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=218349280 ingress_bytes=293617664 link_bytes=293617664 token_bytes=436469472 weight_movement_bytes=75497472 total_nvlink_bytes=511966944 weight_to_token_ratio=0.172973 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=28672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.234091167e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61584 payload_rows_remote=53428 dup_rows=3952 dup_rows_remote=1172 dedup_ratio=0.060303 router_tokens_per_expert=[0, 4053, 0, 2252, 0, 0, 4096, 5, 3624, 18, 0, 0, 4037, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 894, 0, 51, 0, 0, 0, 0, 0, 0, 4090, 7, 0, 0, 0, 0, 0, 0, 1415, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4514, 8076, 8192, 8082, 8180, 8156, 8192, 8192] dup_rows_per_dst=[0, 34, 12, 1056, 14, 2780, 56, 0] combine_rows_per_src=[4514, 8076, 8192, 8082, 8180, 8156, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61384 payload_rows_remote=53204 dup_rows=4152 dup_rows_remote=3166 dedup_ratio=0.063354 router_tokens_per_expert=[0, 4088, 0, 2650, 0, 0, 4095, 0, 3966, 334, 0, 0, 3855, 0, 0, 0, 4, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4090, 493, 0, 0, 0, 0, 0, 0, 783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6158, 7864, 8192, 7932, 8180, 6676, 8192, 8190] dup_rows_per_dst=[660, 514, 10, 2, 986, 1548, 432, 0] combine_rows_per_src=[6158, 7864, 8192, 7932, 8180, 6676, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60720 payload_rows_remote=52612 dup_rows=4816 dup_rows_remote=4802 dedup_ratio=0.073486 router_tokens_per_expert=[5, 4048, 1, 2134, 58, 1, 4075, 0, 2531, 25, 0, 0, 4031, 4, 49, 0, 7, 2, 0, 4073, 0, 0, 0, 4052, 6, 0, 118, 0, 44, 9, 0, 15, 0, 0, 0, 4074, 67, 35, 10, 0, 7, 0, 0, 3143, 38, 0, 9, 0, 9, 1, 0, 0, 0, 13, 0, 53, 0, 8, 5, 0, 0, 8, 0, 0] payload_rows_per_dst=[8176, 8140, 8108, 5362, 8186, 6382, 8182, 8184] dup_rows_per_dst=[4318, 78, 14, 84, 186, 12, 116, 8] combine_rows_per_src=[8176, 8140, 8108, 5362, 8186, 6382, 8182, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56508 payload_rows_remote=48326 dup_rows=9028 dup_rows_remote=8876 dedup_ratio=0.137756 router_tokens_per_expert=[0, 3943, 17, 743, 49, 0, 4078, 0, 3273, 0, 0, 0, 3788, 0, 36, 0, 0, 19, 0, 4081, 0, 0, 0, 3753, 0, 0, 37, 0, 509, 11, 0, 10, 0, 0, 0, 4075, 782, 27, 4, 0, 0, 0, 0, 3440, 0, 0, 8, 0, 0, 5, 0, 55, 0, 10, 0, 0, 0, 0, 5, 0, 2, 0, 8, 0] payload_rows_per_dst=[1524, 7508, 7440, 7338, 8192, 8142, 8182, 8182] dup_rows_per_dst=[94, 0, 0, 482, 1688, 6492, 120, 152] combine_rows_per_src=[1524, 7508, 7440, 7338, 8192, 8142, 8182, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58756 payload_rows_remote=50618 dup_rows=6780 dup_rows_remote=5842 dedup_ratio=0.103455 router_tokens_per_expert=[0, 3844, 0, 1337, 0, 0, 4096, 0, 3973, 355, 0, 0, 4083, 0, 0, 0, 106, 0, 0, 4069, 0, 0, 0, 4096, 0, 0, 529, 0, 36, 0, 0, 0, 0, 0, 0, 3974, 781, 0, 0, 0, 0, 0, 0, 1112, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7702, 8166, 8192, 8138, 7950, 2224, 8192, 8192] dup_rows_per_dst=[2660, 710, 212, 938, 1560, 0, 700, 0] combine_rows_per_src=[7702, 8166, 8192, 8138, 7950, 2224, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58226 payload_rows_remote=50080 dup_rows=7310 dup_rows_remote=7310 dedup_ratio=0.111542 router_tokens_per_expert=[0, 0, 0, 18, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4034, 3719, 3990, 0, 0, 0, 0, 878, 0, 0, 0, 99, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 2754, 27] payload_rows_per_dst=[2218, 7800, 8192, 8070, 7980, 7628, 8146, 8192] dup_rows_per_dst=[0, 0, 56, 0, 0, 1764, 0, 5490] combine_rows_per_src=[2218, 7800, 8192, 8070, 7980, 7628, 8146, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59966 payload_rows_remote=52998 dup_rows=5570 dup_rows_remote=5520 dedup_ratio=0.084991 router_tokens_per_expert=[0, 0, 0, 0, 0, 1221, 20, 0, 0, 0, 0, 0, 28, 36, 0, 0, 719, 1402, 0, 2039, 0, 0, 0, 11, 4082, 0, 0, 529, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 0, 4095, 0, 0, 0, 0, 3445, 0, 4063, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6968, 7396, 7128, 8176, 5764, 8192, 8150] dup_rows_per_dst=[2480, 50, 2386, 654, 0, 0, 0, 0] combine_rows_per_src=[8192, 6968, 7396, 7128, 8176, 5764, 8192, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=218841088 dispatch_meta_bytes=218400 combine_payload_bytes=218841088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=219059488 ingress_bytes=244006912 link_bytes=244006912 token_bytes=437900576 weight_movement_bytes=25165824 total_nvlink_bytes=463066400 weight_to_token_ratio=0.057469 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17408 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.856610666e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55148 payload_rows_remote=46956 dup_rows=10388 dup_rows_remote=3334 dedup_ratio=0.158508 router_tokens_per_expert=[0, 4062, 17, 2274, 58, 0, 4069, 0, 3501, 14, 88, 0, 973, 0, 26, 0, 54, 0, 0, 4079, 0, 0, 0, 4052, 0, 0, 17, 0, 1268, 32, 0, 13, 0, 0, 0, 4033, 83, 18, 16, 1, 0, 10, 0, 3982, 0, 0, 7, 0, 0, 0, 0, 7, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7166, 8158, 8002, 2280, 7990, 5222, 8138] dup_rows_per_dst=[7054, 1014, 608, 1660, 44, 8, 0, 0] combine_rows_per_src=[8192, 7166, 8158, 8002, 2280, 7990, 5222, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=217923584 dispatch_meta_bytes=225480 combine_payload_bytes=217923584 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=218149064 ingress_bytes=268255232 link_bytes=268255232 token_bytes=436072648 weight_movement_bytes=50331648 total_nvlink_bytes=486404296 weight_to_token_ratio=0.115420 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.041112364e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=215498752 dispatch_meta_bytes=229656 combine_payload_bytes=215498752 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=215728408 ingress_bytes=290996224 link_bytes=290996224 token_bytes=431227160 weight_movement_bytes=75497472 total_nvlink_bytes=506724632 weight_to_token_ratio=0.175076 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.214145037e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=197943296 dispatch_meta_bytes=228808 combine_payload_bytes=197943296 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=198172104 ingress_bytes=248274944 link_bytes=248274944 token_bytes=396115400 weight_movement_bytes=50331648 total_nvlink_bytes=446447048 weight_to_token_ratio=0.127063 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.889085458e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=207331328 dispatch_meta_bytes=225840 combine_payload_bytes=207331328 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=207557168 ingress_bytes=257662976 link_bytes=257662976 token_bytes=414888496 weight_movement_bytes=50331648 total_nvlink_bytes=465220144 weight_to_token_ratio=0.121314 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=29056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.960517534e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=205127680 dispatch_meta_bytes=229560 combine_payload_bytes=205127680 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=205357240 ingress_bytes=255459328 link_bytes=255459328 token_bytes=410484920 weight_movement_bytes=50331648 total_nvlink_bytes=460816568 weight_to_token_ratio=0.122615 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.943750319e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=217079808 dispatch_meta_bytes=234072 combine_payload_bytes=217079808 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=217313880 ingress_bytes=267411456 link_bytes=267411456 token_bytes=434393688 weight_movement_bytes=50331648 total_nvlink_bytes=484725336 weight_to_token_ratio=0.115866 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.034692203e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=192331776 dispatch_meta_bytes=201160 combine_payload_bytes=192331776 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=192532936 ingress_bytes=292995072 link_bytes=292995072 token_bytes=384864712 weight_movement_bytes=100663296 total_nvlink_bytes=485528008 weight_to_token_ratio=0.261555 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.229353961e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59142 payload_rows_remote=53998 dup_rows=6394 dup_rows_remote=4590 dedup_ratio=0.097565 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 108, 0, 0, 8, 2919, 0, 1686, 0, 0, 0, 1, 4096, 0, 0, 1444, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 4096, 0, 0, 0, 0, 4025, 0, 4091, 0, 0, 0, 0, 3961, 4078, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8050, 7176, 6320, 8182, 5144, 7922, 8156] dup_rows_per_dst=[0, 222, 2052, 2314, 2, 1804, 0, 0] combine_rows_per_src=[8192, 8050, 7176, 6320, 8182, 5144, 7922, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61310 payload_rows_remote=53122 dup_rows=4226 dup_rows_remote=4222 dedup_ratio=0.064484 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 91, 0, 3, 200, 0, 0, 203, 2170, 0, 880, 1, 0, 0, 0, 4090, 0, 0, 1557, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3291, 0, 4096, 0, 0, 0, 0, 3935, 0, 4094, 0, 0, 0, 0, 4058, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7904, 5948, 8188, 8188, 6582, 8116, 8192] dup_rows_per_dst=[2, 554, 560, 3106, 4, 0, 0, 0] combine_rows_per_src=[8192, 7904, 5948, 8188, 8188, 6582, 8116, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62386 payload_rows_remote=54334 dup_rows=3150 dup_rows_remote=988 dedup_ratio=0.048065 router_tokens_per_expert=[0, 0, 0, 0, 38, 1, 0, 3, 6, 0, 0, 0, 0, 35, 0, 1, 527, 3627, 10, 943, 0, 0, 0, 0, 4066, 23, 6, 172, 8, 0, 0, 2, 17, 77, 0, 118, 10, 0, 6, 0, 2, 3850, 0, 4010, 0, 0, 18, 0, 4076, 0, 4080, 5, 15, 2, 0, 2932, 4021, 0, 0, 2, 8, 18, 16, 17] payload_rows_per_dst=[8092, 8160, 8052, 8166, 8192, 7740, 5904, 8080] dup_rows_per_dst=[12, 76, 2162, 388, 424, 0, 4, 84] combine_rows_per_src=[8092, 8160, 8052, 8166, 8192, 7740, 5904, 8080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61154 payload_rows_remote=52962 dup_rows=4382 dup_rows_remote=882 dedup_ratio=0.066864 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 25, 0, 0, 76, 3477, 0, 511, 0, 0, 0, 9, 4088, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4044, 0, 4090, 0, 0, 0, 0, 3907, 0, 4077, 0, 0, 0, 0, 2559, 4096, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7832, 7384, 8192, 8154, 8090, 5118, 8192] dup_rows_per_dst=[88, 32, 762, 3500, 0, 0, 0, 0] combine_rows_per_src=[8192, 7832, 7384, 8192, 8154, 8090, 5118, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53320 payload_rows_remote=45338 dup_rows=12216 dup_rows_remote=11794 dedup_ratio=0.186401 router_tokens_per_expert=[16, 0, 11, 0, 0, 0, 0, 72, 0, 0, 251, 0, 0, 0, 0, 0, 520, 1312, 9, 635, 0, 14, 0, 0, 4033, 0, 0, 1994, 0, 0, 0, 0, 0, 9, 0, 47, 11, 0, 11, 50, 144, 3610, 0, 3959, 0, 0, 3, 0, 4071, 0, 4071, 3, 18, 3, 0, 3864, 3973, 0, 0, 7, 4, 18, 16, 9] payload_rows_per_dst=[5666, 8146, 4532, 3988, 7334, 7896, 7776, 7982] dup_rows_per_dst=[38, 498, 448, 0, 206, 3642, 6962, 422] combine_rows_per_src=[5666, 8146, 4532, 3988, 7334, 7896, 7776, 7982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62642 payload_rows_remote=54450 dup_rows=2894 dup_rows_remote=2756 dedup_ratio=0.044159 router_tokens_per_expert=[0, 4085, 0, 2812, 0, 0, 4096, 1, 4039, 13, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 39, 0, 38, 0, 0, 0, 0, 0, 0, 4094, 115, 0, 0, 0, 0, 0, 0, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5626, 7978, 8192, 8098, 8188, 8176, 8192, 8192] dup_rows_per_dst=[0, 24, 0, 134, 230, 2368, 138, 0] combine_rows_per_src=[5626, 7978, 8192, 8098, 8188, 8176, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=221175808 dispatch_meta_bytes=234352 combine_payload_bytes=221175808 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=221410160 ingress_bytes=271507456 link_bytes=271507456 token_bytes=442585968 weight_movement_bytes=50331648 total_nvlink_bytes=492917616 weight_to_token_ratio=0.113722 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=25344 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.065858031e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61408 payload_rows_remote=53270 dup_rows=4128 dup_rows_remote=4116 dedup_ratio=0.062988 router_tokens_per_expert=[0, 0, 6, 0, 14, 0, 0, 0, 0, 0, 0, 118, 1, 0, 0, 2, 53, 995, 13, 521, 0, 0, 0, 0, 4029, 18, 0, 3965, 1, 0, 0, 0, 25, 31, 5, 96, 3, 0, 5, 0, 0, 2729, 0, 4055, 0, 0, 0, 0, 3940, 0, 4077, 45, 23, 19, 0, 3929, 3962, 0, 20, 6, 0, 28, 17, 17] payload_rows_per_dst=[8138, 7886, 8086, 7936, 8184, 5458, 7740, 7980] dup_rows_per_dst=[12, 472, 3136, 32, 300, 0, 56, 120] combine_rows_per_src=[8138, 7886, 8086, 7936, 8184, 5458, 7740, 7980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59052 payload_rows_remote=50874 dup_rows=6484 dup_rows_remote=6482 dedup_ratio=0.098938 router_tokens_per_expert=[0, 0, 3, 17, 0, 572, 1, 0, 3108, 0, 0, 4089, 0, 0, 1, 0, 0, 28, 0, 126, 0, 0, 0, 0, 1655, 4096, 45, 0, 0, 0, 0, 3341, 0, 0, 9, 0, 0, 0, 0, 196, 0, 0, 0, 0, 1017, 0, 0, 0, 4096, 0, 683, 4093, 0, 0, 2038, 0, 0, 3547, 0, 0, 7, 0, 0, 0] payload_rows_per_dst=[6620, 8178, 8192, 7572, 8192, 8186, 5018, 7094] dup_rows_per_dst=[782, 2, 308, 2510, 410, 2034, 424, 14] combine_rows_per_src=[6620, 8178, 8192, 7572, 8192, 8186, 5018, 7094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=217587712 dispatch_meta_bytes=229376 combine_payload_bytes=217587712 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=217817088 ingress_bytes=242753536 link_bytes=242753536 token_bytes=435404800 weight_movement_bytes=25165824 total_nvlink_bytes=460570624 weight_to_token_ratio=0.057799 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.847073923e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=222552064 dispatch_meta_bytes=221288 combine_payload_bytes=222552064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=222773352 ingress_bytes=247717888 link_bytes=247717888 token_bytes=445325416 weight_movement_bytes=25165824 total_nvlink_bytes=470491240 weight_to_token_ratio=0.056511 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.884846905e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=216932352 dispatch_meta_bytes=215376 combine_payload_bytes=216932352 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=217147728 ingress_bytes=317595648 link_bytes=317595648 token_bytes=434080080 weight_movement_bytes=100663296 total_nvlink_bytes=534743376 weight_to_token_ratio=0.231900 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.416535920e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=223027200 dispatch_meta_bytes=228824 combine_payload_bytes=223027200 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=223256024 ingress_bytes=248193024 link_bytes=248193024 token_bytes=446283224 weight_movement_bytes=25165824 total_nvlink_bytes=471449048 weight_to_token_ratio=0.056390 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.888462141e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=185704448 dispatch_meta_bytes=228528 combine_payload_bytes=185704448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=185932976 ingress_bytes=210870272 link_bytes=210870272 token_bytes=371637424 weight_movement_bytes=25165824 total_nvlink_bytes=396803248 weight_to_token_ratio=0.067716 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.604479123e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=218193920 dispatch_meta_bytes=229544 combine_payload_bytes=218193920 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=218423464 ingress_bytes=243359744 link_bytes=243359744 token_bytes=436617384 weight_movement_bytes=25165824 total_nvlink_bytes=461783208 weight_to_token_ratio=0.057638 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.851686465e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=208379904 dispatch_meta_bytes=229424 combine_payload_bytes=208379904 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=208609328 ingress_bytes=283877376 link_bytes=283877376 token_bytes=416989232 weight_movement_bytes=75497472 total_nvlink_bytes=492486704 weight_to_token_ratio=0.181054 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.159978829e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56222 payload_rows_remote=48208 dup_rows=9314 dup_rows_remote=9314 dedup_ratio=0.142120 router_tokens_per_expert=[0, 4, 0, 0, 0, 2818, 0, 0, 4086, 30, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 3915, 4096, 106, 0, 0, 0, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 4096, 0, 147, 3995, 0, 0, 819, 0, 0, 432, 0, 0, 21, 0, 0, 0] payload_rows_per_dst=[5712, 8130, 8192, 7980, 8192, 8014, 1912, 8090] dup_rows_per_dst=[1178, 6048, 112, 1150, 0, 0, 20, 806] combine_rows_per_src=[5712, 8130, 8192, 7980, 8192, 8014, 1912, 8090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57688 payload_rows_remote=49496 dup_rows=7848 dup_rows_remote=7726 dedup_ratio=0.119751 router_tokens_per_expert=[0, 34, 0, 5, 0, 2085, 0, 0, 3956, 0, 0, 4052, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 853, 4096, 12, 0, 0, 0, 0, 1640, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 3758, 0, 0, 0, 4096, 0, 1387, 4096, 0, 0, 2224, 0, 0, 407, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8104, 8192, 4786, 8192, 7516, 7962, 4748] dup_rows_per_dst=[3972, 0, 12, 224, 122, 0, 3224, 294] combine_rows_per_src=[8188, 8104, 8192, 4786, 8192, 7516, 7962, 4748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58400 payload_rows_remote=55472 dup_rows=7136 dup_rows_remote=6834 dedup_ratio=0.108887 router_tokens_per_expert=[0, 31, 0, 0, 0, 816, 0, 0, 4008, 5, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 17, 0, 5, 0, 0, 641, 4092, 75, 0, 0, 0, 0, 899, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 3932, 0, 0, 0, 4096, 0, 193, 3933, 0, 0, 2381, 0, 0, 3702, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8092, 7846, 8192, 2928, 8190, 7864, 7884, 7404] dup_rows_per_dst=[1618, 10, 44, 302, 32, 134, 4996, 0] combine_rows_per_src=[8092, 7846, 8192, 2928, 8190, 7864, 7884, 7404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54056 payload_rows_remote=46306 dup_rows=11480 dup_rows_remote=6714 dedup_ratio=0.175171 router_tokens_per_expert=[4, 80, 0, 0, 0, 2659, 0, 21, 3494, 136, 0, 1781, 0, 0, 0, 11, 0, 0, 0, 20, 0, 0, 11, 9, 1973, 3815, 1520, 0, 0, 5, 0, 482, 0, 0, 0, 17, 0, 0, 0, 14, 0, 29, 24, 26, 2096, 0, 1, 3, 4054, 0, 1978, 2304, 0, 0, 3034, 0, 10, 2547, 0, 0, 435, 24, 55, 96] payload_rows_per_dst=[7750, 3738, 8138, 6324, 7488, 7134, 7832, 5652] dup_rows_per_dst=[4766, 118, 50, 1838, 2, 1832, 2192, 682] combine_rows_per_src=[7750, 3738, 8138, 6324, 7488, 7134, 7832, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59354 payload_rows_remote=51608 dup_rows=6182 dup_rows_remote=6182 dedup_ratio=0.094330 router_tokens_per_expert=[0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4, 0, 0, 269, 0, 0, 117, 1554, 0, 1057, 0, 0, 0, 0, 4096, 0, 0, 2812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2606, 0, 4096, 0, 0, 0, 0, 4070, 0, 4095, 0, 0, 0, 0, 3873, 4089, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8150, 5082, 5624, 8190, 8192, 7746, 8178] dup_rows_per_dst=[48, 536, 374, 0, 0, 5224, 0, 0] combine_rows_per_src=[8192, 8150, 5082, 5624, 8190, 8192, 7746, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55520 payload_rows_remote=47328 dup_rows=10016 dup_rows_remote=9690 dedup_ratio=0.152832 router_tokens_per_expert=[2, 0, 0, 0, 4037, 0, 4074, 2791, 0, 0, 4096, 0, 163, 0, 0, 0, 1, 2554, 0, 0, 3999, 3082, 0, 0, 1292, 0, 13, 0, 24, 0, 2043, 0, 403, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 83, 0] payload_rows_per_dst=[8144, 8192, 8170, 6028, 1058, 7774, 7998, 8156] dup_rows_per_dst=[5516, 326, 3104, 716, 186, 0, 6, 162] combine_rows_per_src=[8144, 8192, 8170, 6028, 1058, 7774, 7998, 8156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52726 payload_rows_remote=44676 dup_rows=12810 dup_rows_remote=11162 dedup_ratio=0.195465 router_tokens_per_expert=[0, 0, 38, 0, 0, 974, 0, 21, 3252, 0, 0, 2439, 0, 21, 0, 25, 3, 0, 0, 11, 0, 0, 0, 2, 1162, 4047, 887, 0, 0, 0, 18, 2465, 9, 0, 196, 1, 0, 0, 0, 0, 0, 16, 19, 19, 2681, 0, 0, 3, 4063, 0, 2346, 3898, 0, 0, 3201, 0, 0, 697, 22, 0, 147, 14, 70, 1] payload_rows_per_dst=[2006, 6570, 7524, 7030, 8168, 5400, 7978, 8050] dup_rows_per_dst=[60, 26, 22, 2034, 338, 76, 8606, 1648] combine_rows_per_src=[2006, 6570, 7524, 7030, 8168, 5400, 7978, 8050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53754 payload_rows_remote=45598 dup_rows=11782 dup_rows_remote=11572 dedup_ratio=0.179779 router_tokens_per_expert=[0, 288, 14, 0, 0, 985, 7, 16, 3286, 0, 0, 3374, 0, 31, 0, 0, 0, 0, 0, 99, 1, 0, 70, 5, 3881, 3804, 221, 0, 0, 1, 53, 981, 9, 0, 189, 0, 0, 0, 0, 0, 0, 14, 15, 20, 3403, 0, 22, 14, 4008, 0, 24, 3937, 0, 0, 876, 0, 0, 2637, 0, 0, 310, 15, 8, 150] payload_rows_per_dst=[7188, 6810, 8156, 8032, 7956, 7932, 1792, 5888] dup_rows_per_dst=[2004, 0, 210, 2242, 48, 6918, 8, 352] combine_rows_per_src=[7188, 6810, 8156, 8032, 7956, 7932, 1792, 5888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=197459968 dispatch_meta_bytes=230088 combine_payload_bytes=197459968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197690056 ingress_bytes=222625792 link_bytes=222625792 token_bytes=395150024 weight_movement_bytes=25165824 total_nvlink_bytes=420315848 weight_to_token_ratio=0.063687 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.693925047e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=202735616 dispatch_meta_bytes=228888 combine_payload_bytes=202735616 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202964504 ingress_bytes=227901440 link_bytes=227901440 token_bytes=405700120 weight_movement_bytes=25165824 total_nvlink_bytes=430865944 weight_to_token_ratio=0.062031 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.734066633e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=227213312 dispatch_meta_bytes=249224 combine_payload_bytes=227213312 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=227462536 ingress_bytes=277544960 link_bytes=277544960 token_bytes=454675848 weight_movement_bytes=50331648 total_nvlink_bytes=505007496 weight_to_token_ratio=0.110698 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.111796460e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=189669376 dispatch_meta_bytes=212080 combine_payload_bytes=189669376 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189881456 ingress_bytes=214835200 link_bytes=214835200 token_bytes=379550832 weight_movement_bytes=25165824 total_nvlink_bytes=404716656 weight_to_token_ratio=0.066304 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.634647644e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=211386368 dispatch_meta_bytes=231160 combine_payload_bytes=211386368 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=211617528 ingress_bytes=312049664 link_bytes=312049664 token_bytes=423003896 weight_movement_bytes=100663296 total_nvlink_bytes=523667192 weight_to_token_ratio=0.237973 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=29184 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.374337390e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=193855488 dispatch_meta_bytes=228072 combine_payload_bytes=193855488 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=194083560 ingress_bytes=219021312 link_bytes=219021312 token_bytes=387939048 weight_movement_bytes=25165824 total_nvlink_bytes=413104872 weight_to_token_ratio=0.064871 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.666499119e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=182992896 dispatch_meta_bytes=223352 combine_payload_bytes=182992896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183216248 ingress_bytes=208158720 link_bytes=208158720 token_bytes=366209144 weight_movement_bytes=25165824 total_nvlink_bytes=391374968 weight_to_token_ratio=0.068720 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.583847345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=186769408 dispatch_meta_bytes=228680 combine_payload_bytes=186769408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186998088 ingress_bytes=211935232 link_bytes=211935232 token_bytes=373767496 weight_movement_bytes=25165824 total_nvlink_bytes=398933320 weight_to_token_ratio=0.067330 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.612582238e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54940 payload_rows_remote=46818 dup_rows=10596 dup_rows_remote=10596 dedup_ratio=0.161682 router_tokens_per_expert=[0, 0, 0, 0, 4096, 3, 4095, 3261, 0, 0, 4096, 0, 850, 0, 0, 0, 0, 3, 0, 0, 4096, 3431, 0, 0, 1755, 0, 0, 0, 0, 0, 360, 0, 2426, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 16, 0] payload_rows_per_dst=[1058, 8192, 7556, 5436, 8192, 8122, 8192, 8192] dup_rows_per_dst=[0, 1700, 2110, 1466, 5284, 0, 0, 36] combine_rows_per_src=[1058, 8192, 7556, 5436, 8192, 8122, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59638 payload_rows_remote=51836 dup_rows=5898 dup_rows_remote=2034 dedup_ratio=0.089996 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4094, 2608, 0, 0, 4096, 0, 31, 0, 1, 0, 1, 51, 0, 0, 4043, 3606, 0, 0, 2716, 0, 0, 0, 0, 0, 930, 0, 2420, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5976, 8192, 7270, 6656, 7802, 7468, 8086, 8188] dup_rows_per_dst=[1288, 64, 46, 636, 3864, 0, 0, 0] combine_rows_per_src=[5976, 8192, 7270, 6656, 7802, 7468, 8086, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55600 payload_rows_remote=53586 dup_rows=9936 dup_rows_remote=9936 dedup_ratio=0.151611 router_tokens_per_expert=[24, 0, 0, 0, 4096, 0, 4088, 3777, 0, 0, 4096, 0, 61, 0, 0, 0, 0, 1004, 0, 0, 4076, 3988, 0, 0, 1007, 0, 0, 0, 0, 0, 0, 0, 3781, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 9, 0] payload_rows_per_dst=[8192, 8192, 8094, 2014, 7562, 5204, 8154, 8188] dup_rows_per_dst=[7602, 122, 1890, 0, 306, 0, 10, 6] combine_rows_per_src=[8192, 8192, 8094, 2014, 7562, 5204, 8154, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49868 payload_rows_remote=41750 dup_rows=15668 dup_rows_remote=6618 dedup_ratio=0.239075 router_tokens_per_expert=[0, 0, 0, 0, 3898, 0, 4057, 3168, 0, 0, 4059, 26, 85, 25, 0, 0, 0, 516, 28, 0, 4074, 4048, 0, 0, 4038, 0, 0, 0, 0, 13, 65, 1, 403, 15, 1, 0, 26, 0, 0, 30, 0, 0, 0, 4019, 0, 10, 36, 40, 0, 15, 35, 0, 29, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8118, 5292, 8154, 8138, 950, 8066, 3022, 8128] dup_rows_per_dst=[9050, 62, 6226, 96, 0, 144, 88, 2] combine_rows_per_src=[8118, 5292, 8154, 8138, 950, 8066, 3022, 8128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54572 payload_rows_remote=46380 dup_rows=10964 dup_rows_remote=10004 dedup_ratio=0.167297 router_tokens_per_expert=[28, 1131, 0, 3259, 0, 0, 0, 0, 0, 478, 0, 0, 0, 3, 0, 0, 0, 4095, 0, 98, 4032, 0, 0, 0, 0, 30, 0, 0, 43, 0, 3636, 0, 0, 4, 0, 0, 0, 37, 0, 0, 0, 0, 0, 3487, 2, 495, 0, 0, 3685, 0, 5, 0, 0, 0, 0, 0, 0, 4062, 20, 0, 4079, 0, 59, 0] payload_rows_per_dst=[8192, 8192, 8108, 7370, 8140, 7040, 7372, 158] dup_rows_per_dst=[8802, 960, 152, 48, 66, 928, 8, 0] combine_rows_per_src=[8192, 8192, 8108, 7370, 8140, 7040, 7372, 158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=191766528 dispatch_meta_bytes=229656 combine_payload_bytes=191766528 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=191996184 ingress_bytes=216932352 link_bytes=216932352 token_bytes=383762712 weight_movement_bytes=25165824 total_nvlink_bytes=408928536 weight_to_token_ratio=0.065577 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.650604547e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50654 payload_rows_remote=42494 dup_rows=14882 dup_rows_remote=14752 dedup_ratio=0.227081 router_tokens_per_expert=[0, 0, 0, 0, 4024, 0, 4020, 1162, 0, 0, 4067, 7, 40, 16, 0, 0, 0, 903, 2, 0, 4080, 3752, 0, 0, 3459, 0, 183, 0, 9, 16, 155, 36, 1368, 1046, 5, 0, 1, 0, 0, 64, 0, 0, 0, 3947, 0, 51, 21, 133, 0, 16, 21, 0, 21, 0, 0, 76, 0, 19, 5, 0, 0, 0, 43, 0] payload_rows_per_dst=[0, 8166, 1810, 8188, 8074, 8064, 8192, 8160] dup_rows_per_dst=[0, 94, 0, 7032, 4826, 2564, 236, 130] combine_rows_per_src=[0, 8166, 1810, 8188, 8074, 8064, 8192, 8160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51854 payload_rows_remote=47794 dup_rows=13682 dup_rows_remote=12054 dedup_ratio=0.208771 router_tokens_per_expert=[0, 0, 0, 0, 0, 2800, 0, 0, 3787, 29, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2460, 4095, 37, 0, 0, 0, 0, 1663, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1248, 0, 0, 0, 4096, 0, 7, 4078, 0, 0, 1246, 0, 0, 3328, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[5600, 8000, 8192, 7124, 8192, 2496, 4060, 8190] dup_rows_per_dst=[0, 1956, 0, 3460, 14, 0, 1628, 6624] combine_rows_per_src=[5600, 8000, 8192, 7124, 8192, 2496, 4060, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56492 payload_rows_remote=48522 dup_rows=9044 dup_rows_remote=8592 dedup_ratio=0.138000 router_tokens_per_expert=[0, 0, 0, 0, 4015, 0, 3981, 1281, 0, 0, 4036, 19, 298, 18, 0, 0, 0, 515, 16, 0, 4078, 3680, 0, 0, 596, 0, 96, 0, 0, 12, 110, 43, 3379, 113, 6, 7, 0, 0, 0, 56, 0, 0, 0, 3751, 0, 5, 24, 76, 0, 15, 26, 0, 22, 0, 28, 2450, 0, 7, 0, 0, 0, 0, 9, 0] payload_rows_per_dst=[8030, 8142, 7970, 1620, 6856, 7688, 8192, 7994] dup_rows_per_dst=[2562, 600, 452, 94, 266, 24, 5046, 0] combine_rows_per_src=[8030, 8142, 7970, 1620, 6856, 7688, 8192, 7994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=212320256 dispatch_meta_bytes=215480 combine_payload_bytes=212320256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=212535736 ingress_bytes=237486080 link_bytes=237486080 token_bytes=424855992 weight_movement_bytes=25165824 total_nvlink_bytes=450021816 weight_to_token_ratio=0.059234 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=15232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.806994669e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=219488256 dispatch_meta_bytes=254088 combine_payload_bytes=219488256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=219742344 ingress_bytes=244654080 link_bytes=244654080 token_bytes=439230600 weight_movement_bytes=25165824 total_nvlink_bytes=464396424 weight_to_token_ratio=0.057295 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18176 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.861534867e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=171008000 dispatch_meta_bytes=193472 combine_payload_bytes=171008000 prefetch_bytes=25165824 reduce_bytes=125829120 egress_bytes=171201472 ingress_bytes=322002944 link_bytes=322002944 token_bytes=342209472 weight_movement_bytes=150994944 total_nvlink_bytes=493204416 weight_to_token_ratio=0.441235 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17152 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.450070350e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=189972480 dispatch_meta_bytes=225536 combine_payload_bytes=189972480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=190198016 ingress_bytes=215138304 link_bytes=215138304 token_bytes=380170496 weight_movement_bytes=25165824 total_nvlink_bytes=405336320 weight_to_token_ratio=0.066196 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.636953915e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=174055424 dispatch_meta_bytes=228984 combine_payload_bytes=174055424 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=174284408 ingress_bytes=224387072 link_bytes=224387072 token_bytes=348339832 weight_movement_bytes=50331648 total_nvlink_bytes=398671480 weight_to_token_ratio=0.144490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.707326353e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=195764224 dispatch_meta_bytes=239392 combine_payload_bytes=195764224 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=196003616 ingress_bytes=296427520 link_bytes=296427520 token_bytes=391767840 weight_movement_bytes=100663296 total_nvlink_bytes=492431136 weight_to_token_ratio=0.256946 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.255470924e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=198746112 dispatch_meta_bytes=228456 combine_payload_bytes=198746112 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=198974568 ingress_bytes=274243584 link_bytes=274243584 token_bytes=397720680 weight_movement_bytes=75497472 total_nvlink_bytes=473218152 weight_to_token_ratio=0.189825 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=20608 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.086676803e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54746 payload_rows_remote=47076 dup_rows=10790 dup_rows_remote=8214 dedup_ratio=0.164642 router_tokens_per_expert=[3, 3, 0, 655, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 3991, 0, 0, 0, 0, 2, 0, 0, 41, 0, 4070, 0, 0, 238, 0, 0, 0, 2250, 0, 0, 0, 0, 0, 2584, 0, 2539, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 14, 0] payload_rows_per_dst=[1318, 8192, 7986, 8152, 5086, 7670, 8150, 8192] dup_rows_per_dst=[4, 22, 4, 74, 472, 2576, 0, 7638] combine_rows_per_src=[1318, 8192, 7986, 8152, 5086, 7670, 8150, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60360 payload_rows_remote=52184 dup_rows=5176 dup_rows_remote=1462 dedup_ratio=0.078979 router_tokens_per_expert=[1, 854, 0, 1465, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 144, 4029, 0, 0, 0, 0, 17, 0, 0, 15, 0, 3502, 0, 0, 1883, 0, 0, 0, 34, 0, 0, 0, 0, 0, 4095, 0, 285, 0, 0, 4092, 0, 1, 0, 0, 7, 0, 0, 0, 4028, 9, 0, 4094, 0, 112, 0] payload_rows_per_dst=[4226, 8192, 8164, 7034, 8176, 8190, 8186, 8192] dup_rows_per_dst=[418, 6, 182, 34, 3714, 570, 14, 238] combine_rows_per_src=[4226, 8192, 8164, 7034, 8176, 8190, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54008 payload_rows_remote=45836 dup_rows=11528 dup_rows_remote=6190 dedup_ratio=0.175903 router_tokens_per_expert=[0, 352, 15, 2664, 19, 13, 0, 0, 29, 154, 4, 0, 21, 0, 0, 21, 0, 4016, 34, 0, 4059, 0, 401, 0, 9, 26, 0, 19, 61, 0, 3995, 0, 0, 78, 0, 0, 0, 71, 0, 0, 37, 0, 3, 3738, 0, 1271, 20, 0, 3701, 0, 0, 0, 0, 0, 0, 19, 0, 4072, 0, 0, 3692, 0, 144, 10] payload_rows_per_dst=[8172, 8172, 8192, 8148, 8144, 5432, 7440, 308] dup_rows_per_dst=[5338, 318, 796, 2574, 298, 2204, 0, 0] combine_rows_per_src=[8172, 8172, 8192, 8148, 8144, 5432, 7440, 308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59126 payload_rows_remote=50952 dup_rows=6410 dup_rows_remote=6348 dedup_ratio=0.097809 router_tokens_per_expert=[10, 1806, 0, 575, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 236, 4001, 0, 0, 0, 0, 2, 0, 0, 32, 0, 4084, 0, 0, 281, 0, 0, 0, 103, 0, 0, 0, 0, 0, 3945, 0, 591, 0, 0, 4090, 0, 0, 0, 0, 6, 0, 0, 0, 4042, 1, 0, 4096, 0, 709, 0] payload_rows_per_dst=[6052, 8192, 8178, 8174, 8086, 8034, 8180, 4230] dup_rows_per_dst=[2960, 124, 296, 62, 766, 1038, 12, 1152] combine_rows_per_src=[6052, 8192, 8178, 8174, 8086, 8034, 8180, 4230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=192823296 dispatch_meta_bytes=221160 combine_payload_bytes=192823296 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=193044456 ingress_bytes=268320768 link_bytes=268320768 token_bytes=385867752 weight_movement_bytes=75497472 total_nvlink_bytes=461365224 weight_to_token_ratio=0.195656 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.041611017e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55802 payload_rows_remote=47700 dup_rows=9734 dup_rows_remote=9734 dedup_ratio=0.148529 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4095, 0, 4088, 0, 0, 2021, 2457, 9, 5, 0, 0, 0, 480, 0, 0, 0, 0, 0, 0, 28, 235, 0, 2542, 0, 0, 116, 0, 4051, 4095, 426, 0, 0, 0, 0, 4037, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8150, 8102, 8192, 4930, 2312, 7786, 8140, 8190] dup_rows_per_dst=[0, 0, 6722, 12, 114, 2098, 786, 2] combine_rows_per_src=[8150, 8102, 8192, 4930, 2312, 7786, 8140, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50884 payload_rows_remote=42734 dup_rows=14652 dup_rows_remote=5764 dedup_ratio=0.223572 router_tokens_per_expert=[0, 33, 15, 2051, 5, 19, 0, 0, 26, 334, 7, 10, 0, 0, 0, 25, 0, 4015, 10, 0, 4074, 0, 190, 0, 8, 5, 0, 21, 109, 0, 3839, 0, 16, 7, 0, 0, 0, 2344, 0, 0, 1, 0, 0, 2717, 0, 1044, 47, 6, 4006, 0, 0, 0, 0, 0, 0, 7, 0, 4071, 0, 0, 3556, 0, 146, 4] payload_rows_per_dst=[4208, 5954, 6390, 7922, 4730, 5504, 8026, 8150] dup_rows_per_dst=[38, 676, 376, 42, 4, 1026, 3602, 8888] combine_rows_per_src=[4208, 5954, 6390, 7922, 4730, 5504, 8026, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50936 payload_rows_remote=42744 dup_rows=14600 dup_rows_remote=14478 dedup_ratio=0.222778 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4096, 2319, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 907, 0, 0, 4096, 3777, 0, 0, 3220, 0, 0, 0, 0, 0, 1711, 0, 988, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[0, 8192, 2500, 8184, 8192, 7484, 8192, 8192] dup_rows_per_dst=[0, 8, 184, 9232, 2118, 2934, 122, 2] combine_rows_per_src=[0, 8192, 2500, 8184, 8192, 7484, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=213745664 dispatch_meta_bytes=214584 combine_payload_bytes=213745664 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213960248 ingress_bytes=238911488 link_bytes=238911488 token_bytes=427705912 weight_movement_bytes=25165824 total_nvlink_bytes=452871736 weight_to_token_ratio=0.058839 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.817840377e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54916 payload_rows_remote=46744 dup_rows=10620 dup_rows_remote=10244 dedup_ratio=0.162048 router_tokens_per_expert=[0, 2215, 18, 3225, 15, 52, 0, 1, 22, 10, 8, 4, 2, 0, 0, 27, 0, 4019, 36, 0, 4050, 0, 188, 0, 5, 292, 0, 19, 0, 0, 3625, 0, 0, 33, 14, 0, 0, 1541, 0, 0, 2, 0, 0, 3206, 0, 82, 38, 5, 3877, 0, 0, 0, 0, 0, 0, 6, 0, 4060, 0, 0, 1997, 0, 65, 9] payload_rows_per_dst=[8180, 8092, 8172, 7822, 8152, 6584, 7766, 148] dup_rows_per_dst=[6866, 92, 376, 60, 3144, 82, 0, 0] combine_rows_per_src=[8180, 8092, 8172, 7822, 8152, 6584, 7766, 148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=187744256 dispatch_meta_bytes=208104 combine_payload_bytes=187744256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=187952360 ingress_bytes=212910080 link_bytes=212910080 token_bytes=375696616 weight_movement_bytes=25165824 total_nvlink_bytes=400862440 weight_to_token_ratio=0.066984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=14336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.619999705e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=208699392 dispatch_meta_bytes=229200 combine_payload_bytes=208699392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=208928592 ingress_bytes=233865216 link_bytes=233865216 token_bytes=417627984 weight_movement_bytes=25165824 total_nvlink_bytes=442793808 weight_to_token_ratio=0.060259 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.779444077e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=195379200 dispatch_meta_bytes=229736 combine_payload_bytes=195379200 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=195608936 ingress_bytes=245710848 link_bytes=245710848 token_bytes=390988136 weight_movement_bytes=50331648 total_nvlink_bytes=441319784 weight_to_token_ratio=0.128729 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.869575650e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=175038464 dispatch_meta_bytes=193992 combine_payload_bytes=175038464 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=175232456 ingress_bytes=250535936 link_bytes=250535936 token_bytes=350270920 weight_movement_bytes=75497472 total_nvlink_bytes=425768392 weight_to_token_ratio=0.215540 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17920 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.906288995e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=175079424 dispatch_meta_bytes=228888 combine_payload_bytes=175079424 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=175308312 ingress_bytes=200245248 link_bytes=200245248 token_bytes=350387736 weight_movement_bytes=25165824 total_nvlink_bytes=375553560 weight_to_token_ratio=0.071823 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=29952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.523634967e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=191463424 dispatch_meta_bytes=227952 combine_payload_bytes=191463424 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=191691376 ingress_bytes=266960896 link_bytes=266960896 token_bytes=383154800 weight_movement_bytes=75497472 total_nvlink_bytes=458652272 weight_to_token_ratio=0.197042 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.031263962e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62380 payload_rows_remote=54262 dup_rows=3156 dup_rows_remote=2066 dedup_ratio=0.048157 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 4026, 2816, 0, 0, 0, 0, 0, 947, 0, 0, 0, 0, 0, 0, 0, 437, 0, 77, 0, 0, 64, 0, 4089, 4096, 174, 0, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8178, 8174, 5632, 8192, 8118, 7712, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1894, 1090, 172, 0] combine_rows_per_src=[8182, 8178, 8174, 5632, 8192, 8118, 7712, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62170 payload_rows_remote=53978 dup_rows=3366 dup_rows_remote=2860 dedup_ratio=0.051361 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 4096, 0, 0, 3043, 2846, 1, 0, 0, 0, 0, 247, 0, 0, 0, 6, 0, 0, 0, 45, 0, 2277, 0, 0, 87, 0, 4031, 4095, 49, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8122, 8062, 7668, 5692, 8192, 8076, 8168, 8190] dup_rows_per_dst=[0, 0, 0, 2, 506, 2828, 30, 0] combine_rows_per_src=[8122, 8062, 7668, 5692, 8192, 8076, 8168, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62462 payload_rows_remote=55364 dup_rows=3074 dup_rows_remote=3074 dedup_ratio=0.046906 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4042, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 0, 4096, 0, 0, 3165, 3549, 0, 0, 0, 0, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 133, 0, 719, 0, 0, 26, 0, 4096, 4096, 82, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8108, 8192, 7390, 7098, 8192, 7170, 8120, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2084, 916, 74, 0] combine_rows_per_src=[8108, 8192, 7390, 7098, 8192, 7170, 8120, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50638 payload_rows_remote=42446 dup_rows=14898 dup_rows_remote=14786 dedup_ratio=0.227325 router_tokens_per_expert=[24, 5, 0, 0, 0, 30, 4072, 21, 0, 0, 0, 0, 0, 36, 22, 0, 0, 0, 0, 12, 2, 4025, 0, 4061, 68, 5, 3995, 1065, 0, 0, 32, 21, 46, 917, 0, 5, 1, 388, 13, 26, 1, 0, 0, 343, 0, 0, 205, 0, 3924, 4065, 1987, 0, 0, 0, 0, 3338, 0, 0, 0, 0, 13, 0, 0, 0] payload_rows_per_dst=[8192, 7932, 8150, 6328, 2592, 3298, 6008, 8138] dup_rows_per_dst=[112, 1872, 9998, 1454, 200, 390, 854, 18] combine_rows_per_src=[8192, 7932, 8150, 6328, 2592, 3298, 6008, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=222257152 dispatch_meta_bytes=225312 combine_payload_bytes=222257152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=222482464 ingress_bytes=247422976 link_bytes=247422976 token_bytes=444739616 weight_movement_bytes=25165824 total_nvlink_bytes=469905440 weight_to_token_ratio=0.056586 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.882602966e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58784 payload_rows_remote=50664 dup_rows=6752 dup_rows_remote=1650 dedup_ratio=0.103027 router_tokens_per_expert=[0, 0, 0, 5, 58, 1613, 0, 0, 3353, 0, 0, 1, 0, 0, 0, 11, 0, 4070, 0, 32, 3715, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 117, 4070, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 3898, 0, 4082, 2757, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7824, 8120, 7490, 8168, 3030, 7796, 8192, 8164] dup_rows_per_dst=[1042, 5102, 4, 232, 352, 0, 20, 0] combine_rows_per_src=[7824, 8120, 7490, 8168, 3030, 7796, 8192, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55054 payload_rows_remote=46924 dup_rows=10482 dup_rows_remote=10330 dedup_ratio=0.159943 router_tokens_per_expert=[17, 17, 0, 202, 0, 27, 4067, 0, 0, 0, 0, 0, 26, 25, 16, 0, 0, 0, 3, 11, 0, 3856, 0, 4071, 0, 3, 2863, 3771, 0, 0, 32, 16, 37, 101, 0, 2, 0, 721, 20, 19, 45, 11, 0, 311, 0, 0, 542, 0, 3711, 4063, 187, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[7920, 7514, 22, 8146, 8192, 7166, 7964, 8130] dup_rows_per_dst=[468, 42, 6, 7210, 1750, 378, 476, 152] combine_rows_per_src=[7920, 7514, 22, 8146, 8192, 7166, 7964, 8130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53996 payload_rows_remote=47216 dup_rows=11540 dup_rows_remote=11540 dedup_ratio=0.176086 router_tokens_per_expert=[2, 35, 0, 344, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4079, 0, 0, 0, 0, 24, 0, 0, 1, 0, 3985, 0, 0, 1652, 0, 0, 0, 1627, 0, 0, 0, 0, 0, 3947, 0, 382, 0, 0, 3390, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 7, 0, 4081, 0, 1000, 0] payload_rows_per_dst=[730, 8192, 8164, 7972, 6028, 7942, 6780, 8188] dup_rows_per_dst=[32, 160, 0, 48, 530, 716, 0, 10054] combine_rows_per_src=[730, 8192, 8164, 7972, 6028, 7942, 6780, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=221093888 dispatch_meta_bytes=227352 combine_payload_bytes=221093888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=221321240 ingress_bytes=246259712 link_bytes=246259712 token_bytes=442415128 weight_movement_bytes=25165824 total_nvlink_bytes=467580952 weight_to_token_ratio=0.056883 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.873751871e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59788 payload_rows_remote=51910 dup_rows=5748 dup_rows_remote=5732 dedup_ratio=0.087708 router_tokens_per_expert=[9, 5, 0, 0, 0, 24, 4067, 0, 0, 0, 0, 0, 16, 25, 18, 0, 0, 0, 0, 3, 2, 3921, 21, 4069, 22, 7, 3703, 2222, 0, 0, 24, 20, 0, 905, 2, 28, 15, 333, 21, 18, 8, 624, 0, 960, 0, 0, 0, 0, 3697, 4069, 248, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 7468, 7878, 4542, 8192, 7964, 7418, 8138] dup_rows_per_dst=[22, 44, 16, 48, 2590, 2626, 402, 0] combine_rows_per_src=[8188, 7468, 7878, 4542, 8192, 7964, 7418, 8138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=173858816 dispatch_meta_bytes=228928 combine_payload_bytes=173858816 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=174087744 ingress_bytes=199024640 link_bytes=199024640 token_bytes=347946560 weight_movement_bytes=25165824 total_nvlink_bytes=373112384 weight_to_token_ratio=0.072327 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.514347550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=226770944 dispatch_meta_bytes=233752 combine_payload_bytes=226770944 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=227004696 ingress_bytes=277102592 link_bytes=277102592 token_bytes=453775640 weight_movement_bytes=50331648 total_nvlink_bytes=504107288 weight_to_token_ratio=0.110917 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.108430551e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=207519744 dispatch_meta_bytes=209256 combine_payload_bytes=207519744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=207729000 ingress_bytes=232685568 link_bytes=232685568 token_bytes=415248744 weight_movement_bytes=25165824 total_nvlink_bytes=440414568 weight_to_token_ratio=0.060604 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.770468319e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=192200704 dispatch_meta_bytes=229016 combine_payload_bytes=192200704 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=192429720 ingress_bytes=242532352 link_bytes=242532352 token_bytes=384630424 weight_movement_bytes=50331648 total_nvlink_bytes=434962072 weight_to_token_ratio=0.130857 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.845390968e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=193396736 dispatch_meta_bytes=235024 combine_payload_bytes=193396736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193631760 ingress_bytes=218562560 link_bytes=218562560 token_bytes=387028496 weight_movement_bytes=25165824 total_nvlink_bytes=412194320 weight_to_token_ratio=0.065023 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.663008547e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63850 payload_rows_remote=56944 dup_rows=1686 dup_rows_remote=1686 dedup_ratio=0.025726 router_tokens_per_expert=[0, 0, 0, 0, 0, 63, 0, 0, 3958, 0, 0, 248, 0, 2, 0, 0, 0, 4093, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 4096, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 3453, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7938, 8054, 8192, 8192, 6906, 8192, 8186] dup_rows_per_dst=[122, 478, 0, 268, 818, 0, 0, 0] combine_rows_per_src=[8190, 7938, 8054, 8192, 8192, 6906, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=212623360 dispatch_meta_bytes=230568 combine_payload_bytes=212623360 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=212853928 ingress_bytes=288120832 link_bytes=288120832 token_bytes=425477288 weight_movement_bytes=75497472 total_nvlink_bytes=500974760 weight_to_token_ratio=0.177442 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.192266626e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59192 payload_rows_remote=51006 dup_rows=6344 dup_rows_remote=102 dedup_ratio=0.096802 router_tokens_per_expert=[0, 0, 0, 0, 2, 41, 0, 0, 3093, 4, 0, 7, 0, 0, 0, 0, 0, 4057, 0, 3, 4022, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 4092, 0, 0, 0, 3122, 0, 0, 0, 0, 0, 0, 2128, 0, 4094, 3998, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8016, 6188, 8050, 8116, 8186, 4256, 8192, 8188] dup_rows_per_dst=[66, 20, 0, 16, 6242, 0, 0, 0] combine_rows_per_src=[8016, 6188, 8050, 8116, 8186, 4256, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233242624 dispatch_meta_bytes=234520 combine_payload_bytes=233242624 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=233477144 ingress_bytes=308740096 link_bytes=308740096 token_bytes=466719768 weight_movement_bytes=75497472 total_nvlink_bytes=542217240 weight_to_token_ratio=0.161762 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22400 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.349155401e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52700 payload_rows_remote=44640 dup_rows=12836 dup_rows_remote=12164 dedup_ratio=0.195862 router_tokens_per_expert=[0, 0, 0, 0, 352, 4, 0, 3, 1938, 0, 12, 0, 0, 0, 0, 0, 0, 4067, 0, 1, 4050, 15, 0, 0, 0, 0, 0, 0, 0, 29, 16, 1614, 4063, 0, 0, 18, 738, 317, 0, 12, 0, 0, 25, 3194, 0, 4065, 4007, 25, 0, 13, 1, 0, 12, 36, 4053, 0, 31, 26, 1, 24, 0, 0, 6, 0] payload_rows_per_dst=[8060, 8150, 8152, 3406, 2164, 6720, 8134, 7914] dup_rows_per_dst=[672, 3876, 7976, 50, 6, 48, 96, 112] combine_rows_per_src=[8060, 8150, 8152, 3406, 2164, 6720, 8134, 7914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60994 payload_rows_remote=52866 dup_rows=4542 dup_rows_remote=4510 dedup_ratio=0.069305 router_tokens_per_expert=[0, 0, 0, 0, 2, 325, 0, 0, 3996, 0, 0, 7, 0, 0, 0, 0, 0, 4064, 0, 20, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4092, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 2238, 0, 4096, 4023, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8170, 7992, 7656, 8128, 8192, 4476, 8188, 8192] dup_rows_per_dst=[530, 14, 0, 32, 3966, 0, 0, 0] combine_rows_per_src=[8170, 7992, 7656, 8128, 8192, 4476, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60564 payload_rows_remote=52372 dup_rows=4972 dup_rows_remote=4972 dedup_ratio=0.075867 router_tokens_per_expert=[0, 1787, 3791, 0, 3, 28, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 17, 0, 0, 4077, 622, 0, 197, 0, 0, 0, 0, 51, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0] payload_rows_per_dst=[8004, 8192, 3464, 8192, 8168, 8176, 8192, 8176] dup_rows_per_dst=[3214, 0, 0, 0, 0, 1616, 142, 0] combine_rows_per_src=[8004, 8192, 3464, 8192, 8168, 8176, 8192, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50494 payload_rows_remote=42302 dup_rows=15042 dup_rows_remote=14932 dedup_ratio=0.229523 router_tokens_per_expert=[0, 0, 0, 0, 859, 2, 0, 6, 2900, 0, 10, 2, 0, 17, 0, 0, 0, 4070, 0, 7, 3938, 13, 0, 0, 0, 0, 0, 0, 0, 24, 10, 673, 4067, 0, 0, 29, 796, 312, 0, 5, 0, 0, 17, 2761, 0, 4070, 4064, 15, 0, 12, 0, 0, 0, 1, 3992, 15, 26, 1, 19, 19, 0, 0, 15, 1] payload_rows_per_dst=[3002, 5854, 4622, 8192, 4424, 8168, 8040, 8192] dup_rows_per_dst=[1672, 4, 0, 1362, 1106, 10740, 48, 110] combine_rows_per_src=[3002, 5854, 4622, 8192, 4424, 8168, 8040, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=208920576 dispatch_meta_bytes=204432 combine_payload_bytes=208920576 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=209125008 ingress_bytes=284418048 link_bytes=284418048 token_bytes=418045584 weight_movement_bytes=75497472 total_nvlink_bytes=493543056 weight_to_token_ratio=0.180596 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.164092718e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55516 payload_rows_remote=47330 dup_rows=10020 dup_rows_remote=8994 dedup_ratio=0.152893 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 4096, 0, 0, 732, 3836, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 4, 0, 0, 2, 4, 0, 3, 0, 0, 2075, 0, 4065, 4096, 535, 0, 0, 0, 0, 4071, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8072, 8130, 1320, 7964, 8192, 5460, 8186, 8192] dup_rows_per_dst=[0, 0, 0, 6478, 2342, 172, 1026, 2] combine_rows_per_src=[8072, 8130, 1320, 7964, 8192, 5460, 8186, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=182845440 dispatch_meta_bytes=227216 combine_payload_bytes=182845440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183072656 ingress_bytes=208011264 link_bytes=208011264 token_bytes=365918096 weight_movement_bytes=25165824 total_nvlink_bytes=391083920 weight_to_token_ratio=0.068774 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28544 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.582725375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=216539136 dispatch_meta_bytes=229504 combine_payload_bytes=216539136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=216768640 ingress_bytes=241704960 link_bytes=241704960 token_bytes=433307776 weight_movement_bytes=25165824 total_nvlink_bytes=458473600 weight_to_token_ratio=0.058078 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.839095471e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53172 payload_rows_remote=47338 dup_rows=12364 dup_rows_remote=12364 dedup_ratio=0.188660 router_tokens_per_expert=[0, 0, 0, 0, 885, 105, 0, 15, 1759, 0, 11, 0, 0, 0, 0, 0, 0, 4073, 0, 39, 2875, 3, 0, 0, 0, 0, 0, 0, 0, 23, 6, 1435, 4058, 0, 0, 26, 3660, 39, 0, 12, 0, 0, 26, 1445, 0, 4073, 3989, 20, 0, 67, 0, 0, 0, 16, 4038, 0, 25, 0, 0, 20, 0, 0, 22, 3] payload_rows_per_dst=[8146, 3540, 5834, 8192, 8192, 2942, 8134, 8192] dup_rows_per_dst=[1842, 0, 0, 2882, 7398, 40, 108, 94] combine_rows_per_src=[8146, 3540, 5834, 8192, 8192, 2942, 8134, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60994 payload_rows_remote=52802 dup_rows=4542 dup_rows_remote=4526 dedup_ratio=0.069305 router_tokens_per_expert=[0, 1830, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 3, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[3660, 8192, 8184, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4516, 0, 0, 16, 10, 0] combine_rows_per_src=[3660, 8192, 8184, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=214515712 dispatch_meta_bytes=229376 combine_payload_bytes=214515712 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=214745088 ingress_bytes=290013184 link_bytes=290013184 token_bytes=429260800 weight_movement_bytes=75497472 total_nvlink_bytes=504758272 weight_to_token_ratio=0.175878 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.206665239e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=173268992 dispatch_meta_bytes=228936 combine_payload_bytes=173268992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=173497928 ingress_bytes=198434816 link_bytes=198434816 token_bytes=346766920 weight_movement_bytes=25165824 total_nvlink_bytes=371932744 weight_to_token_ratio=0.072573 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.509859671e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=193863680 dispatch_meta_bytes=225296 combine_payload_bytes=193863680 prefetch_bytes=25165824 reduce_bytes=125829120 egress_bytes=194088976 ingress_bytes=344858624 link_bytes=344858624 token_bytes=387952656 weight_movement_bytes=150994944 total_nvlink_bytes=538947600 weight_to_token_ratio=0.389210 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29056 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.623975667e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=193896448 dispatch_meta_bytes=238808 combine_payload_bytes=193896448 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=194135256 ingress_bytes=269393920 link_bytes=269393920 token_bytes=388031704 weight_movement_bytes=75497472 total_nvlink_bytes=463529176 weight_to_token_ratio=0.194565 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.049776464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=216276992 dispatch_meta_bytes=229312 combine_payload_bytes=216276992 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=216506304 ingress_bytes=266608640 link_bytes=266608640 token_bytes=432783296 weight_movement_bytes=50331648 total_nvlink_bytes=483114944 weight_to_token_ratio=0.116298 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.028583701e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6219446776 prefetch_bytes=402653184 reduce_bytes=377487360 weight_movement_bytes=780140544 egress_bytes=3111541240 ingress_bytes=3888046080 link_bytes=3888046080 total_nvlink_bytes=6999587320 weight_to_token_ratio=0.125436 prefetch_share_of_ingress=0.103562 dedup_ratio=0.162619 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=15 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=16256 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.848971142e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63860 payload_rows_remote=55668 dup_rows=1676 dup_rows_remote=1672 dedup_ratio=0.025574 router_tokens_per_expert=[0, 3325, 4030, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 610, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4095, 32, 0, 0, 0, 0, 0, 0, 112, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0] payload_rows_per_dst=[6652, 8192, 8076, 8192, 8192, 8190, 8192, 8174] dup_rows_per_dst=[0, 0, 1218, 0, 4, 64, 390, 0] combine_rows_per_src=[6652, 8192, 8076, 8192, 8192, 8190, 8192, 8174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57536 payload_rows_remote=49358 dup_rows=8000 dup_rows_remote=1388 dedup_ratio=0.122070 router_tokens_per_expert=[0, 1183, 4060, 0, 61, 0, 22, 3, 4017, 0, 34, 0, 0, 2, 4000, 0, 0, 1, 0, 0, 1697, 8, 1094, 0, 0, 15, 27, 0, 0, 11, 0, 0, 17, 0, 0, 4047, 56, 5, 44, 0, 2, 0, 4057, 14, 29, 1, 12, 0, 15, 0, 250, 0, 70, 3, 18, 0, 0, 15, 0, 0, 0, 3823, 4055, 0] payload_rows_per_dst=[8178, 8166, 5600, 8088, 7758, 5300, 8170, 6276] dup_rows_per_dst=[6612, 420, 0, 52, 248, 16, 652, 0] combine_rows_per_src=[8178, 8166, 5600, 8088, 7758, 5300, 8170, 6276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59810 payload_rows_remote=51618 dup_rows=5726 dup_rows_remote=5726 dedup_ratio=0.087372 router_tokens_per_expert=[0, 2220, 3950, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 7, 0, 0, 4096, 251, 0, 143, 0, 0, 0, 0, 68, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0] payload_rows_per_dst=[5750, 8192, 4934, 8192, 8192, 8192, 8192, 8166] dup_rows_per_dst=[2432, 0, 2222, 0, 10, 788, 274, 0] combine_rows_per_src=[5750, 8192, 4934, 8192, 8192, 8192, 8192, 8166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6411151840 prefetch_bytes=402653184 reduce_bytes=427819008 weight_movement_bytes=830472192 egress_bytes=3207391712 ingress_bytes=4034232320 link_bytes=4034232320 total_nvlink_bytes=7241624032 weight_to_token_ratio=0.129536 prefetch_share_of_ingress=0.099809 dedup_ratio=0.143251 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=17 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=16002 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.918490416e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228016128 dispatch_meta_bytes=229360 combine_payload_bytes=228016128 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=228245488 ingress_bytes=278347776 link_bytes=278347776 token_bytes=456261616 weight_movement_bytes=50331648 total_nvlink_bytes=506593264 weight_to_token_ratio=0.110313 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.117904962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57440 payload_rows_remote=49646 dup_rows=8096 dup_rows_remote=5608 dedup_ratio=0.123535 router_tokens_per_expert=[0, 1703, 4072, 0, 0, 0, 0, 10, 4047, 0, 45, 0, 0, 4, 3992, 0, 0, 0, 0, 0, 2362, 7, 6, 0, 1, 1, 23, 0, 0, 15, 0, 0, 13, 31, 0, 4042, 28, 9, 41, 0, 1, 0, 4070, 40, 23, 1, 0, 12, 0, 0, 191, 0, 3, 11, 5, 0, 0, 0, 0, 0, 0, 3897, 4062, 0] payload_rows_per_dst=[3426, 7882, 8162, 8140, 8176, 8188, 5672, 7794] dup_rows_per_dst=[0, 0, 4732, 234, 152, 106, 384, 2488] combine_rows_per_src=[3426, 7882, 8162, 8140, 8176, 8188, 5672, 7794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=202170368 dispatch_meta_bytes=202984 combine_payload_bytes=202170368 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=202373352 ingress_bytes=252502016 link_bytes=252502016 token_bytes=404543720 weight_movement_bytes=50331648 total_nvlink_bytes=454875368 weight_to_token_ratio=0.124416 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.921248592e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=211427328 dispatch_meta_bytes=229376 combine_payload_bytes=211427328 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=211656704 ingress_bytes=261758976 link_bytes=261758976 token_bytes=423084032 weight_movement_bytes=50331648 total_nvlink_bytes=473415680 weight_to_token_ratio=0.118964 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.991683362e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59184 payload_rows_remote=50992 dup_rows=6352 dup_rows_remote=6352 dedup_ratio=0.096924 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 4078, 0, 0, 52, 0, 0, 0, 0, 0, 4096, 0, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4096, 0, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 3212, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 8168, 7698, 8192, 2690, 7872, 8192, 8192] dup_rows_per_dst=[2, 92, 0, 108, 578, 5572, 0, 0] combine_rows_per_src=[8180, 8168, 7698, 8192, 2690, 7872, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58490 payload_rows_remote=56924 dup_rows=7046 dup_rows_remote=7046 dedup_ratio=0.107513 router_tokens_per_expert=[0, 3173, 3851, 0, 31, 0, 2, 25, 3987, 0, 22, 0, 0, 0, 4032, 0, 0, 6, 0, 0, 765, 7, 5, 0, 0, 0, 24, 0, 0, 20, 0, 0, 19, 0, 0, 4048, 22, 3, 36, 0, 27, 0, 4063, 106, 23, 0, 2, 0, 1, 0, 319, 0, 13, 11, 0, 16, 0, 15, 0, 0, 0, 4024, 4070, 0] payload_rows_per_dst=[8190, 8108, 1566, 8022, 8178, 8178, 8186, 8062] dup_rows_per_dst=[5974, 0, 0, 40, 78, 264, 674, 16] combine_rows_per_src=[8190, 8108, 1566, 8022, 8178, 8178, 8186, 8062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6523624248 prefetch_bytes=377487360 reduce_bytes=427819008 weight_movement_bytes=805306368 egress_bytes=3263617848 ingress_bytes=4065312768 link_bytes=4065312768 total_nvlink_bytes=7328930616 weight_to_token_ratio=0.123445 prefetch_share_of_ingress=0.092856 dedup_ratio=0.125977 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=17 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=17145 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.933270810e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5835892592 prefetch_bytes=402653184 reduce_bytes=276824064 weight_movement_bytes=679477248 egress_bytes=2919663472 ingress_bytes=3595706368 link_bytes=3595706368 total_nvlink_bytes=6515369840 weight_to_token_ratio=0.116431 prefetch_share_of_ingress=0.111982 dedup_ratio=0.204348 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=11 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=17399 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.709948178e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=203350016 dispatch_meta_bytes=221016 combine_payload_bytes=203350016 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=203571032 ingress_bytes=278847488 link_bytes=278847488 token_bytes=406921048 weight_movement_bytes=75497472 total_nvlink_bytes=482418520 weight_to_token_ratio=0.185533 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.121707193e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6602136024 prefetch_bytes=478150656 reduce_bytes=452984832 weight_movement_bytes=931135488 egress_bytes=3302889944 ingress_bytes=4230381568 link_bytes=4230381568 total_nvlink_bytes=7533271512 weight_to_token_ratio=0.141035 prefetch_share_of_ingress=0.113028 dedup_ratio=0.125235 prefetch_slots=38 unused_prefetch_slots=218 slot_utilization=0.148438 reduce_slots=18 remote_demand=19 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=17399 gemm_flops_step=21028159881216 bytes_per_gemm_flop=2.011769737e-04 prefetch_bytes_per_gemm_flop=2.273858762e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=208863232 dispatch_meta_bytes=229376 combine_payload_bytes=208863232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=209092608 ingress_bytes=234029056 link_bytes=234029056 token_bytes=417955840 weight_movement_bytes=25165824 total_nvlink_bytes=443121664 weight_to_token_ratio=0.060212 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.780690711e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=233160704 dispatch_meta_bytes=255880 combine_payload_bytes=233160704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=233416584 ingress_bytes=258326528 link_bytes=258326528 token_bytes=466577288 weight_movement_bytes=25165824 total_nvlink_bytes=491743112 weight_to_token_ratio=0.053937 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18432 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.965566398e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5678410368 prefetch_bytes=402653184 reduce_bytes=301989888 weight_movement_bytes=704643072 egress_bytes=2840988288 ingress_bytes=3542065152 link_bytes=3542065152 total_nvlink_bytes=6383053440 weight_to_token_ratio=0.124092 prefetch_share_of_ingress=0.113678 dedup_ratio=0.225344 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=12 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039752 gemm_inner_pad_upper_bound=16383 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.684438949e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6202678656 prefetch_bytes=352321536 reduce_bytes=452984832 weight_movement_bytes=805306368 egress_bytes=3103137152 ingress_bytes=3904847872 link_bytes=3904847872 total_nvlink_bytes=7007985024 weight_to_token_ratio=0.129832 prefetch_share_of_ingress=0.090227 dedup_ratio=0.165684 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=18 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=17145 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.856961281e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60786 payload_rows_remote=52594 dup_rows=4750 dup_rows_remote=4728 dedup_ratio=0.072479 router_tokens_per_expert=[0, 1722, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[3444, 8192, 8192, 8192, 8190, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 4670, 0, 0, 58, 22, 0] combine_rows_per_src=[3444, 8192, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=215425024 dispatch_meta_bytes=229288 combine_payload_bytes=215425024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=215654312 ingress_bytes=240590848 link_bytes=240590848 token_bytes=431079336 weight_movement_bytes=25165824 total_nvlink_bytes=456245160 weight_to_token_ratio=0.058379 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.830618366e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6056658184 prefetch_bytes=427819008 reduce_bytes=528482304 weight_movement_bytes=956301312 egress_bytes=3030115592 ingress_bytes=3982843904 link_bytes=3982843904 total_nvlink_bytes=7012959496 weight_to_token_ratio=0.157893 prefetch_share_of_ingress=0.107415 dedup_ratio=0.180141 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=21 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042509 gemm_inner_pad_upper_bound=16129 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.894052512e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none + [2026-09-21 08:31:46] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 226.7 | throughput per GPU (TFLOP/s/GPU): 142.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.641895E+00 | loss scale: 1.0 | grad norm: 3.471 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45834 payload_rows_remote=39838 dup_rows=19702 dup_rows_remote=16858 dedup_ratio=0.300629 router_tokens_per_expert=[120, 287, 1707, 0, 1, 594, 720, 619, 4, 897, 98, 1, 13, 9, 510, 172, 2, 905, 601, 1285, 40, 149, 249, 7, 78, 19, 135, 370, 574, 718, 946, 1580, 1183, 735, 546, 912, 204, 613, 548, 673, 1658, 880, 1763, 101, 756, 692, 11, 109, 823, 927, 696, 34, 234, 1600, 1343, 32, 922, 18, 288, 1, 3, 22, 997, 34] payload_rows_per_dst=[5808, 5984, 5018, 5996, 5534, 6290, 5804, 5400] dup_rows_per_dst=[2288, 2840, 1458, 2844, 2292, 3236, 2888, 1856] combine_rows_per_src=[5808, 5984, 5018, 5996, 5534, 6290, 5804, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45404 payload_rows_remote=38918 dup_rows=20132 dup_rows_remote=16500 dedup_ratio=0.307190 router_tokens_per_expert=[293, 340, 1126, 2, 0, 926, 682, 309, 1, 1057, 42, 3, 4, 38, 569, 353, 2, 668, 1184, 1321, 39, 256, 300, 8, 136, 19, 144, 518, 1034, 916, 1028, 1528, 617, 541, 658, 1531, 216, 844, 1058, 800, 1531, 403, 1105, 176, 771, 536, 4, 20, 723, 648, 741, 74, 121, 1005, 1465, 151, 1138, 43, 229, 4, 10, 18, 716, 25] payload_rows_per_dst=[6452, 6486, 6338, 6088, 5956, 3360, 5062, 5662] dup_rows_per_dst=[3966, 3632, 3046, 2730, 2800, 460, 1864, 1634] combine_rows_per_src=[6452, 6486, 6338, 6088, 5956, 3360, 5062, 5662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46010 payload_rows_remote=40596 dup_rows=19526 dup_rows_remote=17418 dedup_ratio=0.297943 router_tokens_per_expert=[140, 360, 1117, 3, 0, 597, 593, 433, 2, 464, 36, 1, 13, 39, 485, 173, 7, 1002, 928, 1360, 45, 157, 354, 7, 137, 11, 126, 337, 972, 565, 1424, 1476, 735, 848, 836, 939, 242, 737, 745, 1036, 1596, 718, 1359, 108, 985, 1021, 14, 34, 1021, 838, 481, 37, 156, 1306, 1472, 48, 1044, 22, 258, 0, 8, 9, 727, 24] payload_rows_per_dst=[4980, 5214, 5838, 6496, 5414, 7030, 5492, 5546] dup_rows_per_dst=[1506, 1926, 1882, 3600, 2108, 4640, 2282, 1582] combine_rows_per_src=[4980, 5214, 5838, 6496, 5414, 7030, 5492, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45982 payload_rows_remote=39736 dup_rows=19554 dup_rows_remote=16574 dedup_ratio=0.298370 router_tokens_per_expert=[175, 267, 1303, 0, 0, 674, 966, 409, 3, 352, 50, 0, 13, 4, 523, 143, 12, 1513, 821, 1163, 104, 106, 284, 2, 154, 42, 164, 258, 787, 601, 1111, 1714, 937, 425, 486, 1296, 249, 973, 792, 532, 1575, 311, 1392, 72, 578, 546, 21, 118, 773, 1000, 850, 39, 279, 1642, 1647, 141, 866, 47, 442, 21, 30, 36, 908, 26] payload_rows_per_dst=[5418, 5130, 5924, 5224, 5114, 6246, 6338, 6588] dup_rows_per_dst=[2170, 1584, 2086, 1776, 1728, 2980, 3556, 3674] combine_rows_per_src=[5418, 5130, 5924, 5224, 5114, 6246, 6338, 6588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45952 payload_rows_remote=39362 dup_rows=19584 dup_rows_remote=16144 dedup_ratio=0.298828 router_tokens_per_expert=[244, 359, 894, 2, 2, 673, 976, 534, 2, 461, 27, 1, 2, 56, 581, 292, 7, 455, 844, 1268, 55, 410, 546, 13, 195, 35, 160, 507, 1043, 907, 1153, 1495, 668, 753, 512, 1429, 180, 1170, 837, 683, 1214, 559, 1347, 31, 903, 652, 9, 78, 860, 694, 553, 107, 266, 619, 1994, 49, 996, 25, 285, 3, 97, 11, 941, 44] payload_rows_per_dst=[5270, 5610, 5566, 5478, 5360, 6440, 5638, 6590] dup_rows_per_dst=[2098, 2432, 1630, 2132, 1906, 3536, 2410, 3440] combine_rows_per_src=[5270, 5610, 5566, 5478, 5360, 6440, 5638, 6590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45962 payload_rows_remote=39990 dup_rows=19574 dup_rows_remote=16842 dedup_ratio=0.298676 router_tokens_per_expert=[72, 340, 1565, 2, 2, 759, 672, 770, 0, 427, 54, 2, 39, 11, 827, 87, 0, 805, 687, 1909, 24, 353, 162, 10, 165, 37, 85, 196, 1039, 772, 813, 1360, 769, 438, 595, 856, 181, 1184, 745, 590, 1598, 664, 2079, 80, 389, 597, 23, 126, 876, 979, 781, 33, 236, 1395, 1371, 87, 754, 46, 574, 3, 66, 59, 533, 15] payload_rows_per_dst=[5798, 5026, 5954, 4870, 5078, 7052, 5972, 6212] dup_rows_per_dst=[2566, 1948, 1946, 1344, 1558, 4060, 2732, 3420] combine_rows_per_src=[5798, 5026, 5954, 4870, 5078, 7052, 5972, 6212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 6, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 3, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163176448 dispatch_meta_bytes=226784 combine_payload_bytes=163176448 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=163403232 ingress_bytes=238673920 link_bytes=238673920 token_bytes=326579680 weight_movement_bytes=75497472 total_nvlink_bytes=402077152 weight_to_token_ratio=0.231176 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.816032759e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=221672 combine_payload_bytes=159408128 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=159629800 ingress_bytes=234905600 link_bytes=234905600 token_bytes=319037928 weight_movement_bytes=75497472 total_nvlink_bytes=394535400 weight_to_token_ratio=0.236641 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=9344 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787360198e-04 prefetch_bytes_per_gemm_flop=5.744485294e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=166281216 dispatch_meta_bytes=232056 combine_payload_bytes=166281216 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=166513272 ingress_bytes=266944512 link_bytes=266944512 token_bytes=332794488 weight_movement_bytes=100663296 total_nvlink_bytes=433457784 weight_to_token_ratio=0.302479 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=6528 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.031139299e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=162758656 dispatch_meta_bytes=225240 combine_payload_bytes=162758656 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=162983896 ingress_bytes=238256128 link_bytes=238256128 token_bytes=325742552 weight_movement_bytes=75497472 total_nvlink_bytes=401240024 weight_to_token_ratio=0.231770 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.812853844e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=45976 payload_rows_remote=40002 dup_rows=19560 dup_rows_remote=17670 dedup_ratio=0.298462 router_tokens_per_expert=[159, 228, 1241, 3, 0, 501, 612, 455, 0, 405, 13, 2, 33, 10, 899, 138, 10, 1298, 1027, 1097, 53, 224, 220, 3, 117, 49, 86, 296, 1032, 868, 1166, 1884, 985, 393, 567, 1108, 319, 894, 735, 1014, 1229, 629, 1399, 103, 833, 574, 17, 87, 776, 979, 703, 40, 382, 1085, 1146, 37, 1054, 38, 311, 46, 60, 116, 962, 18] payload_rows_per_dst=[5672, 6010, 5974, 6810, 5894, 4406, 5558, 5652] dup_rows_per_dst=[2476, 2964, 1890, 4186, 2960, 788, 2446, 1850] combine_rows_per_src=[5672, 6010, 5974, 6810, 5894, 4406, 5558, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=161226752 dispatch_meta_bytes=222024 combine_payload_bytes=161226752 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=161448776 ingress_bytes=211558400 link_bytes=211558400 token_bytes=322675528 weight_movement_bytes=50331648 total_nvlink_bytes=373007176 weight_to_token_ratio=0.155982 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=11648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.609714982e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44928 payload_rows_remote=38976 dup_rows=20608 dup_rows_remote=17492 dedup_ratio=0.314453 router_tokens_per_expert=[156, 252, 1013, 2, 0, 596, 753, 534, 3, 666, 37, 0, 15, 40, 416, 146, 6, 1004, 513, 1414, 30, 411, 345, 4, 58, 7, 70, 527, 1072, 487, 760, 1766, 1114, 1001, 766, 1191, 202, 747, 812, 1082, 1228, 714, 2054, 144, 763, 860, 6, 19, 780, 862, 750, 36, 152, 1147, 1183, 20, 801, 13, 338, 14, 9, 34, 810, 13] payload_rows_per_dst=[5952, 5936, 6770, 4622, 6948, 4308, 5390, 5002] dup_rows_per_dst=[3116, 2814, 4216, 1340, 4886, 704, 2104, 1428] combine_rows_per_src=[5952, 5936, 6770, 4622, 6948, 4308, 5390, 5002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163799040 dispatch_meta_bytes=227328 combine_payload_bytes=163799040 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=164026368 ingress_bytes=239296512 link_bytes=239296512 token_bytes=327825408 weight_movement_bytes=75497472 total_nvlink_bytes=403322880 weight_to_token_ratio=0.230298 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=7040 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.820769964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=163848192 dispatch_meta_bytes=230688 combine_payload_bytes=163848192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=164078880 ingress_bytes=189014016 link_bytes=189014016 token_bytes=327927072 weight_movement_bytes=25165824 total_nvlink_bytes=353092896 weight_to_token_ratio=0.076742 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.438178268e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48432 payload_rows_remote=42790 dup_rows=17104 dup_rows_remote=15156 dedup_ratio=0.260986 router_tokens_per_expert=[3019, 2, 4, 0, 0, 238, 35, 2809, 359, 3665, 15, 1838, 0, 0, 1436, 0, 22, 0, 233, 3, 406, 595, 830, 2, 0, 1732, 0, 0, 432, 1711, 296, 0, 0, 0, 1, 1, 251, 46, 494, 7, 41, 0, 764, 0, 0, 0, 313, 16, 70, 0, 78, 3786, 2802, 0, 1, 0, 2283, 4, 386, 1660, 48, 0, 34, 0] payload_rows_per_dst=[6682, 5284, 5830, 5642, 4694, 7796, 5754, 6750] dup_rows_per_dst=[6718, 1810, 1774, 1948, 582, 2044, 148, 2080] combine_rows_per_src=[6682, 5284, 5830, 5642, 4694, 7796, 5754, 6750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48676 payload_rows_remote=43502 dup_rows=16860 dup_rows_remote=15704 dedup_ratio=0.257263 router_tokens_per_expert=[1715, 0, 18, 0, 0, 709, 0, 1701, 36, 1127, 14, 3471, 0, 0, 2368, 0, 2, 0, 50, 106, 473, 852, 257, 0, 0, 1264, 0, 0, 1803, 1784, 139, 110, 0, 0, 0, 0, 179, 2, 817, 0, 82, 0, 719, 0, 0, 0, 662, 0, 168, 1, 1153, 3389, 3464, 0, 58, 0, 3323, 0, 497, 249, 6, 0, 0, 0] payload_rows_per_dst=[4406, 5982, 3814, 7228, 7360, 5174, 7636, 7076] dup_rows_per_dst=[478, 1108, 752, 5288, 1578, 1156, 5426, 1074] combine_rows_per_src=[4406, 5982, 3814, 7228, 7360, 5174, 7636, 7076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51302 payload_rows_remote=43708 dup_rows=14234 dup_rows_remote=11090 dedup_ratio=0.217194 router_tokens_per_expert=[1559, 0, 304, 0, 0, 46, 2, 637, 85, 1244, 7, 3561, 0, 0, 3472, 7, 1, 0, 35, 856, 527, 146, 456, 0, 1, 2289, 0, 0, 255, 3600, 90, 1, 0, 0, 3, 2, 663, 34, 40, 4, 0, 0, 1223, 0, 0, 0, 95, 0, 51, 0, 1336, 3204, 3457, 0, 1, 0, 2775, 1, 515, 128, 12, 0, 43, 0] payload_rows_per_dst=[5878, 7594, 7708, 4956, 7606, 7126, 7356, 3078] dup_rows_per_dst=[1706, 3144, 3534, 316, 1008, 1918, 2334, 274] combine_rows_per_src=[5878, 7594, 7708, 4956, 7606, 7126, 7356, 3078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=225872 combine_payload_bytes=159645696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159871568 ingress_bytes=184811520 link_bytes=184811520 token_bytes=319517264 weight_movement_bytes=25165824 total_nvlink_bytes=344683088 weight_to_token_ratio=0.078762 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406202129e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50858 payload_rows_remote=43158 dup_rows=14678 dup_rows_remote=13322 dedup_ratio=0.223969 router_tokens_per_expert=[1807, 0, 4, 0, 0, 129, 1, 1807, 884, 3350, 78, 3767, 0, 0, 2028, 2, 0, 0, 7, 478, 803, 158, 6, 1, 0, 1867, 0, 0, 123, 3587, 185, 0, 1, 0, 0, 0, 603, 142, 15, 0, 61, 1, 1175, 0, 0, 0, 486, 0, 10, 1, 106, 2613, 2408, 0, 146, 0, 1743, 2, 434, 1602, 70, 0, 77, 0] payload_rows_per_dst=[3766, 7828, 7548, 6170, 7700, 6742, 4986, 6118] dup_rows_per_dst=[116, 4856, 2532, 1794, 1356, 1930, 356, 1738] combine_rows_per_src=[3766, 7828, 7548, 6170, 7700, 6742, 4986, 6118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46028 payload_rows_remote=38054 dup_rows=19508 dup_rows_remote=12610 dedup_ratio=0.297668 router_tokens_per_expert=[2521, 1, 3, 0, 1, 10, 0, 1608, 6, 1634, 1, 3007, 0, 0, 3266, 0, 5, 0, 1, 73, 5, 1034, 29, 1, 0, 1824, 0, 0, 229, 2046, 0, 17, 13, 0, 24, 0, 40, 514, 427, 0, 30, 1, 1807, 0, 0, 0, 460, 0, 133, 0, 1212, 3321, 661, 0, 28, 0, 3376, 0, 984, 2204, 20, 0, 191, 0] payload_rows_per_dst=[5046, 5154, 2250, 7244, 7048, 4256, 7056, 7974] dup_rows_per_dst=[26, 152, 46, 4204, 5510, 340, 2332, 6898] combine_rows_per_src=[5046, 5154, 2250, 7244, 7048, 4256, 7056, 7974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49484 payload_rows_remote=41918 dup_rows=16052 dup_rows_remote=12746 dedup_ratio=0.244934 router_tokens_per_expert=[1681, 0, 18, 0, 2, 298, 0, 3467, 509, 3903, 139, 3817, 0, 0, 983, 32, 0, 9, 48, 313, 3709, 631, 72, 0, 0, 772, 0, 0, 548, 1586, 129, 4, 0, 0, 2, 0, 57, 21, 13, 0, 104, 0, 226, 0, 0, 0, 16, 0, 0, 32, 1228, 1007, 3587, 0, 14, 0, 1903, 0, 941, 443, 241, 0, 263, 0] payload_rows_per_dst=[3866, 8022, 7668, 7824, 7662, 692, 7566, 6184] dup_rows_per_dst=[132, 3110, 1896, 5188, 158, 0, 3306, 2262] combine_rows_per_src=[3866, 8022, 7668, 7824, 7662, 692, 7566, 6184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=175267840 dispatch_meta_bytes=231784 combine_payload_bytes=175267840 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=175499624 ingress_bytes=225599488 link_bytes=225599488 token_bytes=350767464 weight_movement_bytes=50331648 total_nvlink_bytes=401099112 weight_to_token_ratio=0.143490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=12032 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.716551438e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=178184192 dispatch_meta_bytes=236824 combine_payload_bytes=178184192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=178421016 ingress_bytes=203350016 link_bytes=203350016 token_bytes=356605208 weight_movement_bytes=25165824 total_nvlink_bytes=381771032 weight_to_token_ratio=0.070571 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18432 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.547258664e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=179027968 dispatch_meta_bytes=219192 combine_payload_bytes=179027968 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=179247160 ingress_bytes=279691264 link_bytes=279691264 token_bytes=358275128 weight_movement_bytes=100663296 total_nvlink_bytes=458938424 weight_to_token_ratio=0.280966 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=14080 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.128127354e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=176775168 dispatch_meta_bytes=225920 combine_payload_bytes=176775168 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=177001088 ingress_bytes=227106816 link_bytes=227106816 token_bytes=353776256 weight_movement_bytes=50331648 total_nvlink_bytes=404107904 weight_to_token_ratio=0.142270 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.728020462e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=155869184 dispatch_meta_bytes=202656 combine_payload_bytes=155869184 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=156071840 ingress_bytes=206200832 link_bytes=206200832 token_bytes=311941024 weight_movement_bytes=50331648 total_nvlink_bytes=362272672 weight_to_token_ratio=0.161350 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.568950080e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=171696128 dispatch_meta_bytes=218656 combine_payload_bytes=171696128 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=171914784 ingress_bytes=222027776 link_bytes=222027776 token_bytes=343610912 weight_movement_bytes=50331648 total_nvlink_bytes=393942560 weight_to_token_ratio=0.146479 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=20224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.689374836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49018 payload_rows_remote=41774 dup_rows=16518 dup_rows_remote=13532 dedup_ratio=0.252045 router_tokens_per_expert=[1647, 0, 2, 0, 0, 658, 2, 3654, 957, 1851, 284, 3466, 0, 0, 1008, 3, 0, 0, 10, 1586, 479, 495, 43, 0, 0, 1278, 0, 0, 565, 2502, 404, 32, 0, 0, 0, 0, 324, 22, 123, 1, 0, 2, 516, 1, 0, 0, 136, 0, 2, 3, 383, 3190, 2633, 0, 96, 0, 2739, 0, 1571, 40, 18, 0, 41, 1] payload_rows_per_dst=[8102, 3190, 7244, 3876, 7338, 6800, 5830, 6638] dup_rows_per_dst=[7526, 1314, 2986, 682, 534, 890, 404, 2182] combine_rows_per_src=[8102, 3190, 7244, 3876, 7338, 6800, 5830, 6638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46546 payload_rows_remote=38868 dup_rows=18990 dup_rows_remote=15580 dedup_ratio=0.289764 router_tokens_per_expert=[1104, 0, 1, 0, 6, 132, 3, 3310, 741, 988, 350, 2455, 0, 0, 1653, 0, 0, 0, 180, 6, 1740, 254, 15, 0, 0, 908, 0, 0, 678, 2782, 2, 0, 0, 0, 0, 0, 101, 4, 141, 0, 51, 0, 2252, 0, 0, 0, 3195, 1, 15, 0, 322, 3837, 2179, 0, 7, 0, 750, 0, 2272, 178, 76, 0, 79, 0] payload_rows_per_dst=[7678, 5656, 7190, 2934, 5304, 8152, 4790, 4842] dup_rows_per_dst=[3410, 1332, 2764, 242, 98, 10520, 256, 368] combine_rows_per_src=[7678, 5656, 7190, 2934, 5304, 8152, 4790, 4842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52594 payload_rows_remote=47276 dup_rows=12942 dup_rows_remote=12084 dedup_ratio=0.197479 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 12, 0, 1203, 0, 14, 2, 0, 643, 0, 232, 53, 5, 3369, 0, 11, 0, 0, 0, 0, 0, 0, 3764, 0, 2, 0, 0, 830, 1591, 696, 0, 761, 2287, 3079, 0, 0, 0, 0, 0, 105, 0, 0, 0, 126, 0, 2775, 46, 1, 42, 1, 0, 11, 4050, 0, 0, 470, 3405, 3179, 0] payload_rows_per_dst=[8100, 3680, 6772, 7532, 7584, 5318, 5682, 7926] dup_rows_per_dst=[30, 508, 104, 0, 4938, 858, 300, 6204] combine_rows_per_src=[8100, 3680, 6772, 7532, 7584, 5318, 5682, 7926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=41700 payload_rows_remote=33572 dup_rows=23836 dup_rows_remote=16620 dedup_ratio=0.363708 router_tokens_per_expert=[0, 1, 58, 0, 77, 5, 6, 0, 2872, 0, 0, 0, 14, 54, 1, 1232, 586, 53, 3137, 4, 0, 0, 0, 0, 0, 0, 2, 4015, 0, 2001, 0, 0, 0, 735, 2347, 0, 273, 3128, 126, 0, 0, 0, 0, 2, 155, 0, 0, 0, 299, 113, 3802, 62, 0, 30, 192, 0, 2, 1428, 0, 0, 210, 2247, 3499, 0] payload_rows_per_dst=[3012, 8128, 6766, 8182, 7494, 4840, 2854, 424] dup_rows_per_dst=[138, 7216, 1046, 11078, 3754, 168, 436, 0] combine_rows_per_src=[3012, 8128, 6766, 8182, 7494, 4840, 2854, 424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49164 payload_rows_remote=46976 dup_rows=16372 dup_rows_remote=16372 dedup_ratio=0.249817 router_tokens_per_expert=[0, 53, 0, 0, 115, 0, 1254, 0, 1170, 0, 8, 0, 0, 161, 0, 92, 47, 261, 2231, 6, 22, 0, 0, 1, 2, 0, 0, 1087, 4, 0, 1, 0, 75, 474, 4015, 0, 1537, 2366, 408, 0, 0, 4, 0, 23, 108, 0, 0, 0, 1867, 0, 3892, 235, 1, 1, 0, 0, 0, 4082, 0, 6, 258, 2953, 3948, 0] payload_rows_per_dst=[8188, 4936, 5096, 2188, 5572, 8134, 7960, 7090] dup_rows_per_dst=[2820, 1462, 292, 0, 2016, 2046, 4032, 3704] combine_rows_per_src=[8188, 4936, 5096, 2188, 5572, 8134, 7960, 7090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=171106304 dispatch_meta_bytes=221224 combine_payload_bytes=171106304 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=171327528 ingress_bytes=196272128 link_bytes=196272128 token_bytes=342433832 weight_movement_bytes=25165824 total_nvlink_bytes=367599656 weight_to_token_ratio=0.073491 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.493404114e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48626 payload_rows_remote=41562 dup_rows=16910 dup_rows_remote=12804 dedup_ratio=0.258026 router_tokens_per_expert=[0, 0, 4, 0, 0, 0, 21, 0, 2342, 0, 12, 26, 0, 6, 0, 146, 3, 1, 1111, 45, 0, 0, 0, 0, 2, 0, 0, 2942, 93, 170, 0, 1, 1, 1893, 2581, 0, 978, 3426, 2712, 1, 0, 0, 0, 0, 201, 2, 0, 0, 1170, 33, 3571, 187, 9, 158, 1, 0, 0, 2563, 1, 0, 369, 3419, 2567, 0] payload_rows_per_dst=[5150, 4708, 2272, 6134, 7064, 7696, 7864, 7738] dup_rows_per_dst=[26, 356, 48, 282, 4106, 4724, 2394, 4974] combine_rows_per_src=[5150, 4708, 2272, 6134, 7064, 7696, 7864, 7738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159203328 dispatch_meta_bytes=217792 combine_payload_bytes=159203328 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159421120 ingress_bytes=209534976 link_bytes=209534976 token_bytes=318624448 weight_movement_bytes=50331648 total_nvlink_bytes=368956096 weight_to_token_ratio=0.157965 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594319063e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54198 payload_rows_remote=47278 dup_rows=11338 dup_rows_remote=9100 dedup_ratio=0.173004 router_tokens_per_expert=[0, 0, 1, 2, 31, 0, 66, 0, 3736, 0, 1, 110, 0, 486, 1, 357, 2563, 487, 3746, 21, 0, 0, 0, 0, 3, 4, 8, 3806, 0, 25, 0, 0, 0, 65, 1787, 0, 68, 1605, 1301, 2, 0, 0, 0, 11, 131, 1, 0, 0, 432, 0, 3638, 21, 0, 0, 0, 0, 27, 3673, 1, 0, 9, 2082, 2460, 0] payload_rows_per_dst=[7346, 7714, 7990, 7634, 2802, 5952, 7840, 6920] dup_rows_per_dst=[200, 1668, 5644, 58, 70, 1118, 342, 2238] combine_rows_per_src=[7346, 7714, 7990, 7634, 2802, 5952, 7840, 6920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52990 payload_rows_remote=46662 dup_rows=12546 dup_rows_remote=10562 dedup_ratio=0.191437 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 29, 0, 1565, 0, 0, 65, 2, 75, 0, 29, 0, 29, 4030, 43, 3, 0, 0, 0, 0, 0, 0, 2714, 0, 0, 0, 0, 51, 255, 2034, 0, 3109, 1488, 1444, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 2012, 449, 1482, 194, 1, 0, 18, 0, 1, 4035, 0, 0, 977, 3247, 1845, 0] payload_rows_per_dst=[8084, 3298, 8090, 5428, 7166, 7026, 6328, 7570] dup_rows_per_dst=[46, 174, 120, 0, 2552, 3100, 1984, 4570] combine_rows_per_src=[8084, 3298, 8090, 5428, 7166, 7026, 6328, 7570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193642496 dispatch_meta_bytes=237440 combine_payload_bytes=193642496 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=193879936 ingress_bytes=243974144 link_bytes=243974144 token_bytes=387522432 weight_movement_bytes=50331648 total_nvlink_bytes=437854080 weight_to_token_ratio=0.129881 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18944 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.856361339e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=137510912 dispatch_meta_bytes=200768 combine_payload_bytes=137510912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=137711680 ingress_bytes=162676736 link_bytes=162676736 token_bytes=275222592 weight_movement_bytes=25165824 total_nvlink_bytes=300388416 weight_to_token_ratio=0.091438 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.237781998e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=192413696 dispatch_meta_bytes=253392 combine_payload_bytes=192413696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=192667088 ingress_bytes=217579520 link_bytes=217579520 token_bytes=385080784 weight_movement_bytes=25165824 total_nvlink_bytes=410246608 weight_to_token_ratio=0.065352 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.655528748e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=170237952 dispatch_meta_bytes=217464 combine_payload_bytes=170237952 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=170455416 ingress_bytes=270901248 link_bytes=270901248 token_bytes=340693368 weight_movement_bytes=100663296 total_nvlink_bytes=441356664 weight_to_token_ratio=0.295466 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=8832 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.061245488e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=193650688 dispatch_meta_bytes=225512 combine_payload_bytes=193650688 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=193876200 ingress_bytes=319479808 link_bytes=319479808 token_bytes=387526888 weight_movement_bytes=125829120 total_nvlink_bytes=513356008 weight_to_token_ratio=0.324698 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17280 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.430872200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=191127552 dispatch_meta_bytes=228896 combine_payload_bytes=191127552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=191356448 ingress_bytes=241459200 link_bytes=241459200 token_bytes=382484000 weight_movement_bytes=50331648 total_nvlink_bytes=432815648 weight_to_token_ratio=0.131592 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=19328 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.837225521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53292 payload_rows_remote=45486 dup_rows=12244 dup_rows_remote=8708 dedup_ratio=0.186829 router_tokens_per_expert=[0, 0, 174, 1, 1, 0, 223, 0, 1171, 0, 50, 3, 32, 628, 0, 6, 49, 117, 2817, 1, 0, 0, 0, 0, 1, 0, 0, 3094, 2, 1, 0, 0, 10, 856, 3687, 0, 169, 1596, 2687, 0, 0, 0, 0, 0, 320, 0, 0, 0, 1873, 5, 3720, 8, 0, 0, 3, 0, 3, 3873, 0, 0, 902, 2101, 2583, 1] payload_rows_per_dst=[7804, 6662, 7806, 6194, 4496, 7464, 7902, 4964] dup_rows_per_dst=[740, 2284, 3536, 2, 766, 550, 3416, 950] combine_rows_per_src=[7804, 6662, 7806, 6194, 4496, 7464, 7902, 4964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46332 payload_rows_remote=38210 dup_rows=19204 dup_rows_remote=13474 dedup_ratio=0.293030 router_tokens_per_expert=[3, 0, 0, 0, 4, 0, 532, 0, 1630, 0, 2, 3, 0, 17, 0, 903, 600, 299, 1375, 3, 135, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 27, 269, 1786, 0, 408, 1761, 2565, 0, 0, 0, 0, 3, 1482, 0, 0, 0, 2484, 6, 3422, 566, 37, 0, 8, 0, 29, 3935, 0, 0, 103, 2565, 3816, 0] payload_rows_per_dst=[8122, 7940, 6872, 7312, 4418, 5854, 5556, 258] dup_rows_per_dst=[5730, 4802, 3082, 3512, 738, 688, 646, 6] combine_rows_per_src=[8122, 7940, 6872, 7312, 4418, 5854, 5556, 258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=186310656 dispatch_meta_bytes=216776 combine_payload_bytes=186310656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186527432 ingress_bytes=211476480 link_bytes=211476480 token_bytes=372838088 weight_movement_bytes=25165824 total_nvlink_bytes=398003912 weight_to_token_ratio=0.067498 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.609091665e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53648 payload_rows_remote=45678 dup_rows=11888 dup_rows_remote=11698 dedup_ratio=0.181396 router_tokens_per_expert=[304, 0, 0, 0, 0, 2, 1, 0, 0, 146, 1752, 0, 0, 1593, 0, 3647, 0, 0, 602, 897, 0, 3, 25, 0, 0, 0, 6, 3920, 1, 363, 0, 1181, 0, 0, 364, 0, 0, 0, 0, 0, 43, 0, 66, 3549, 0, 0, 0, 3971, 0, 2380, 0, 0, 1688, 0, 39, 0, 0, 0, 0, 0, 4095, 2019, 0, 111] payload_rows_per_dst=[7432, 5348, 7490, 2950, 7842, 7970, 6424, 8192] dup_rows_per_dst=[476, 1634, 2662, 152, 726, 190, 1790, 4258] combine_rows_per_src=[7432, 5348, 7490, 2950, 7842, 7970, 6424, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53642 payload_rows_remote=45514 dup_rows=11894 dup_rows_remote=10746 dedup_ratio=0.181488 router_tokens_per_expert=[1454, 0, 0, 0, 0, 5, 0, 13, 0, 486, 3481, 0, 0, 350, 0, 1557, 0, 0, 1764, 208, 0, 14, 30, 0, 2, 0, 112, 4056, 0, 1699, 0, 2095, 3, 0, 578, 0, 0, 0, 0, 1, 29, 0, 305, 1451, 13, 0, 0, 2552, 0, 3846, 116, 0, 1512, 0, 305, 0, 0, 1, 0, 2, 4088, 635, 0, 5] payload_rows_per_dst=[4892, 7286, 5758, 6146, 8128, 5420, 7834, 8178] dup_rows_per_dst=[1166, 1348, 1176, 1670, 1148, 378, 3724, 1284] combine_rows_per_src=[4892, 7286, 5758, 6146, 8128, 5420, 7834, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49582 payload_rows_remote=42682 dup_rows=15954 dup_rows_remote=13110 dedup_ratio=0.243439 router_tokens_per_expert=[28, 4, 0, 0, 0, 128, 0, 0, 0, 2292, 3923, 0, 0, 567, 2, 2562, 0, 0, 2852, 1214, 4, 3, 623, 0, 0, 0, 574, 3595, 2, 45, 0, 10, 0, 0, 98, 0, 0, 0, 0, 9, 206, 0, 0, 1232, 18, 0, 0, 3756, 2, 2356, 15, 0, 1407, 0, 64, 0, 0, 0, 0, 0, 4078, 1099, 0, 0] payload_rows_per_dst=[7268, 6900, 7456, 7890, 214, 7532, 6490, 5832] dup_rows_per_dst=[2000, 2844, 4400, 562, 0, 2096, 2384, 1668] combine_rows_per_src=[7268, 6900, 7456, 7890, 214, 7532, 6490, 5832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56752 payload_rows_remote=49550 dup_rows=8784 dup_rows_remote=8056 dedup_ratio=0.134033 router_tokens_per_expert=[555, 5, 6, 0, 0, 0, 0, 30, 0, 32, 1727, 0, 0, 817, 3, 4009, 0, 0, 1, 290, 3, 0, 26, 0, 0, 4, 4, 4072, 0, 544, 0, 3413, 0, 0, 1626, 0, 0, 4, 3, 16, 159, 0, 987, 4020, 18, 0, 0, 3012, 0, 504, 0, 0, 2523, 0, 18, 1, 0, 0, 0, 0, 4077, 200, 3, 56] payload_rows_per_dst=[8124, 4598, 8060, 7202, 8182, 6820, 5582, 8184] dup_rows_per_dst=[1086, 560, 620, 728, 3260, 1532, 510, 488] combine_rows_per_src=[8124, 4598, 8060, 7202, 8182, 6820, 5582, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=156508160 dispatch_meta_bytes=206736 combine_payload_bytes=156508160 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=156714896 ingress_bytes=206839808 link_bytes=206839808 token_bytes=313223056 weight_movement_bytes=50331648 total_nvlink_bytes=363554704 weight_to_token_ratio=0.160689 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.573811949e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50880 payload_rows_remote=42688 dup_rows=14656 dup_rows_remote=13426 dedup_ratio=0.223633 router_tokens_per_expert=[61, 0, 0, 0, 0, 568, 0, 4, 0, 325, 3085, 0, 0, 1745, 0, 3205, 0, 0, 2316, 682, 0, 2, 713, 2, 0, 0, 127, 3569, 10, 3732, 0, 453, 3, 0, 468, 0, 0, 4, 0, 2, 5, 0, 45, 1640, 1, 0, 0, 2168, 0, 1564, 68, 0, 928, 0, 562, 0, 0, 0, 0, 0, 4096, 615, 0, 0] payload_rows_per_dst=[6686, 3864, 6188, 8130, 7448, 5820, 4552, 8192] dup_rows_per_dst=[990, 276, 1242, 6684, 644, 1898, 1692, 1230] combine_rows_per_src=[6686, 3864, 6188, 8130, 7448, 5820, 4552, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51062 payload_rows_remote=45912 dup_rows=14474 dup_rows_remote=13822 dedup_ratio=0.220856 router_tokens_per_expert=[0, 0, 0, 0, 0, 557, 1, 5, 0, 2572, 2736, 0, 0, 235, 0, 4096, 0, 0, 271, 90, 0, 0, 24, 0, 0, 0, 202, 3972, 0, 563, 0, 2281, 1, 0, 3855, 0, 0, 0, 0, 0, 17, 0, 6, 2999, 0, 0, 0, 1300, 0, 2197, 0, 0, 290, 0, 414, 0, 0, 0, 0, 0, 4075, 7, 0, 2] payload_rows_per_dst=[8192, 7278, 3370, 6056, 8188, 4678, 5150, 8150] dup_rows_per_dst=[1126, 2550, 122, 1294, 7468, 1244, 652, 18] combine_rows_per_src=[8192, 7278, 3370, 6056, 8188, 4678, 5150, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=187097088 dispatch_meta_bytes=229504 combine_payload_bytes=187097088 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=187326592 ingress_bytes=237428736 link_bytes=237428736 token_bytes=374423680 weight_movement_bytes=50331648 total_nvlink_bytes=424755328 weight_to_token_ratio=0.134424 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=21632 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.806558347e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=186425344 dispatch_meta_bytes=225040 combine_payload_bytes=186425344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186650384 ingress_bytes=211591168 link_bytes=211591168 token_bytes=373075728 weight_movement_bytes=25165824 total_nvlink_bytes=398241552 weight_to_token_ratio=0.067455 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23040 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.609964308e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=174825472 dispatch_meta_bytes=223168 combine_payload_bytes=174825472 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=175048640 ingress_bytes=275488768 link_bytes=275488768 token_bytes=349874112 weight_movement_bytes=100663296 total_nvlink_bytes=450537408 weight_to_token_ratio=0.287713 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=13568 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.096151215e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=202956800 dispatch_meta_bytes=230424 combine_payload_bytes=202956800 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=203187224 ingress_bytes=253288448 link_bytes=253288448 token_bytes=406144024 weight_movement_bytes=50331648 total_nvlink_bytes=456475672 weight_to_token_ratio=0.123926 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.927232431e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=174850048 dispatch_meta_bytes=224456 combine_payload_bytes=174850048 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=175074504 ingress_bytes=225181696 link_bytes=225181696 token_bytes=349924552 weight_movement_bytes=50331648 total_nvlink_bytes=400256200 weight_to_token_ratio=0.143836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=27264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.713372523e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=188055552 dispatch_meta_bytes=238936 combine_payload_bytes=188055552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=188294488 ingress_bytes=238387200 link_bytes=238387200 token_bytes=376350040 weight_movement_bytes=50331648 total_nvlink_bytes=426681688 weight_to_token_ratio=0.133736 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=15616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.813851151e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53456 payload_rows_remote=46138 dup_rows=12080 dup_rows_remote=11710 dedup_ratio=0.184326 router_tokens_per_expert=[220, 0, 0, 0, 1, 21, 0, 0, 0, 479, 2393, 0, 0, 2491, 0, 3629, 0, 0, 9, 181, 0, 2, 7, 0, 0, 0, 332, 3766, 0, 16, 0, 594, 0, 0, 2540, 0, 0, 3, 0, 0, 168, 0, 503, 3645, 0, 0, 0, 3917, 0, 2054, 1, 0, 167, 0, 1314, 0, 0, 23, 0, 0, 4095, 82, 0, 115] payload_rows_per_dst=[7328, 6966, 7318, 2524, 7252, 7988, 5888, 8192] dup_rows_per_dst=[414, 3760, 370, 198, 4528, 1188, 1184, 438] combine_rows_per_src=[7328, 6966, 7318, 2524, 7252, 7988, 5888, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47356 payload_rows_remote=40472 dup_rows=18180 dup_rows_remote=16708 dedup_ratio=0.277405 router_tokens_per_expert=[207, 0, 0, 0, 0, 228, 0, 0, 0, 1698, 479, 0, 0, 2007, 0, 3264, 0, 2, 616, 176, 0, 0, 8, 0, 0, 0, 176, 4096, 0, 88, 0, 2315, 0, 0, 246, 0, 8, 0, 0, 0, 0, 0, 398, 3850, 1, 0, 0, 896, 0, 2117, 23, 0, 3460, 0, 1697, 0, 0, 0, 0, 0, 4096, 615, 0, 1] payload_rows_per_dst=[6884, 6762, 7892, 8192, 500, 8192, 7702, 1232] dup_rows_per_dst=[1472, 3562, 1412, 5158, 8, 2590, 3978, 0] combine_rows_per_src=[6884, 6762, 7892, 8192, 500, 8192, 7702, 1232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=188981248 dispatch_meta_bytes=231392 combine_payload_bytes=188981248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=189212640 ingress_bytes=214147072 link_bytes=214147072 token_bytes=378193888 weight_movement_bytes=25165824 total_nvlink_bytes=403359712 weight_to_token_ratio=0.066542 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.629411785e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54064 payload_rows_remote=48772 dup_rows=11472 dup_rows_remote=10084 dedup_ratio=0.175049 router_tokens_per_expert=[0, 3747, 0, 0, 1734, 0, 0, 68, 3878, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 0, 0, 0, 0, 3317, 0, 0, 49, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4086, 0, 189, 0, 2056, 3549, 0, 0, 0, 1, 0, 6, 0, 0, 0, 3757, 0, 0, 0, 0, 0, 0, 4030] payload_rows_per_dst=[3580, 8186, 6584, 6672, 8172, 5292, 8064, 7514] dup_rows_per_dst=[24, 7098, 2886, 60, 6, 1388, 10, 0] combine_rows_per_src=[3580, 8186, 6584, 6672, 8172, 5292, 8064, 7514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53456 payload_rows_remote=45590 dup_rows=12080 dup_rows_remote=12042 dedup_ratio=0.184326 router_tokens_per_expert=[0, 4094, 0, 0, 2594, 0, 0, 0, 3750, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1583, 0, 0, 0, 0, 582, 0, 0, 1897, 0, 0, 27, 0, 0, 0, 13, 0, 0, 3912, 26, 2483, 0, 3233, 629, 2, 0, 0, 0, 0, 67, 32, 0, 0, 3833, 0, 29, 0, 0, 0, 0, 3976] payload_rows_per_dst=[5188, 8192, 4164, 4670, 7866, 7694, 7960, 7722] dup_rows_per_dst=[0, 7506, 262, 288, 38, 3794, 190, 2] combine_rows_per_src=[5188, 8192, 4164, 4670, 7866, 7694, 7960, 7722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58926 payload_rows_remote=52854 dup_rows=6610 dup_rows_remote=6320 dedup_ratio=0.100861 router_tokens_per_expert=[0, 4090, 0, 15, 1321, 0, 0, 16, 4072, 5, 0, 0, 9, 1, 6, 0, 11, 0, 0, 4, 0, 6, 1, 3423, 1, 0, 0, 0, 2855, 20, 0, 305, 0, 1, 0, 0, 0, 0, 0, 5, 0, 4059, 1, 2642, 0, 704, 1521, 0, 0, 0, 0, 6, 215, 0, 0, 0, 3383, 1, 8, 0, 0, 0, 6, 4055] payload_rows_per_dst=[8072, 8160, 7460, 6072, 8130, 6066, 8174, 6792] dup_rows_per_dst=[2582, 256, 2472, 290, 0, 628, 378, 4] combine_rows_per_src=[8072, 8160, 7460, 6072, 8130, 6066, 8174, 6792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54704 payload_rows_remote=51408 dup_rows=10832 dup_rows_remote=10390 dedup_ratio=0.165283 router_tokens_per_expert=[0, 3914, 0, 1, 2042, 0, 0, 55, 832, 3, 0, 0, 1034, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 2213, 0, 0, 0, 0, 2932, 0, 0, 1349, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2132, 0, 879, 3510, 13, 0, 0, 0, 0, 2, 17, 8, 0, 3651, 0, 134, 0, 0, 0, 0, 3921] payload_rows_per_dst=[8018, 3296, 7914, 7626, 8192, 4268, 7888, 7502] dup_rows_per_dst=[4006, 442, 3588, 2694, 4, 22, 8, 68] combine_rows_per_src=[8018, 3296, 7914, 7626, 8192, 4268, 7888, 7502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=165773312 dispatch_meta_bytes=228720 combine_payload_bytes=165773312 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=166002032 ingress_bytes=216104960 link_bytes=216104960 token_bytes=331775344 weight_movement_bytes=50331648 total_nvlink_bytes=382106992 weight_to_token_ratio=0.151704 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.644309050e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51836 payload_rows_remote=44480 dup_rows=13700 dup_rows_remote=12406 dedup_ratio=0.209045 router_tokens_per_expert=[0, 3676, 6, 14, 3320, 4, 0, 126, 2668, 0, 11, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2195, 0, 7, 0, 0, 3881, 0, 0, 180, 3, 0, 13, 0, 0, 0, 0, 0, 0, 4091, 0, 3244, 0, 2636, 1905, 0, 0, 0, 1, 0, 224, 111, 14, 0, 649, 0, 0, 0, 0, 0, 0, 3785] payload_rows_per_dst=[3548, 5358, 4396, 7830, 7238, 8168, 7942, 7356] dup_rows_per_dst=[70, 2, 0, 306, 26, 11674, 328, 1294] combine_rows_per_src=[3548, 5358, 4396, 7830, 7238, 8168, 7942, 7356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56830 payload_rows_remote=48638 dup_rows=8706 dup_rows_remote=8542 dedup_ratio=0.132843 router_tokens_per_expert=[0, 4060, 0, 0, 2322, 0, 0, 2, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2921, 0, 0, 0, 0, 3278, 0, 0, 52, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 1, 51, 0, 3749, 1628, 0, 0, 0, 0, 2, 70, 10, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096] payload_rows_per_dst=[4646, 7118, 5866, 6612, 8192, 8016, 8192, 8188] dup_rows_per_dst=[2, 1804, 2, 48, 16, 2842, 164, 3828] combine_rows_per_src=[4646, 7118, 5866, 6612, 8192, 8016, 8192, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=199770112 dispatch_meta_bytes=235424 combine_payload_bytes=199770112 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=200005536 ingress_bytes=300433408 link_bytes=300433408 token_bytes=399775648 weight_movement_bytes=100663296 total_nvlink_bytes=500438944 weight_to_token_ratio=0.251799 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=13952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.285951103e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=186736640 dispatch_meta_bytes=230528 combine_payload_bytes=186736640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=186967168 ingress_bytes=211902464 link_bytes=211902464 token_bytes=373703808 weight_movement_bytes=25165824 total_nvlink_bytes=398869632 weight_to_token_ratio=0.067342 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.612332911e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=216489984 dispatch_meta_bytes=236696 combine_payload_bytes=216489984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=216726680 ingress_bytes=241655808 link_bytes=241655808 token_bytes=433216664 weight_movement_bytes=25165824 total_nvlink_bytes=458382488 weight_to_token_ratio=0.058091 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=23552 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.838721481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=210567168 dispatch_meta_bytes=247192 combine_payload_bytes=210567168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210814360 ingress_bytes=235732992 link_bytes=235732992 token_bytes=421381528 weight_movement_bytes=25165824 total_nvlink_bytes=446547352 weight_to_token_ratio=0.059722 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21632 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.793655695e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=182190080 dispatch_meta_bytes=227544 combine_payload_bytes=182190080 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=182417624 ingress_bytes=257687552 link_bytes=257687552 token_bytes=364607704 weight_movement_bytes=75497472 total_nvlink_bytes=440105176 weight_to_token_ratio=0.207065 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.960704529e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=199221248 dispatch_meta_bytes=228720 combine_payload_bytes=199221248 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=199449968 ingress_bytes=249552896 link_bytes=249552896 token_bytes=398671216 weight_movement_bytes=50331648 total_nvlink_bytes=449002864 weight_to_token_ratio=0.126249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.898809196e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55418 payload_rows_remote=47274 dup_rows=10118 dup_rows_remote=7538 dedup_ratio=0.154388 router_tokens_per_expert=[0, 4083, 0, 2, 1779, 0, 0, 9, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1292, 0, 0, 0, 0, 3925, 0, 0, 17, 0, 0, 209, 0, 0, 0, 0, 0, 0, 4095, 0, 1033, 0, 3077, 4058, 2, 0, 0, 0, 53, 10, 3, 29, 0, 4066, 0, 87, 0, 0, 0, 0, 3448] payload_rows_per_dst=[8180, 2958, 8144, 7886, 8192, 4844, 7034, 8180] dup_rows_per_dst=[3566, 0, 2580, 3186, 416, 192, 52, 126] combine_rows_per_src=[8180, 2958, 8144, 7886, 8192, 4844, 7034, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52084 payload_rows_remote=43916 dup_rows=13452 dup_rows_remote=7566 dedup_ratio=0.205261 router_tokens_per_expert=[0, 4083, 0, 0, 2865, 0, 0, 79, 2556, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2000, 0, 0, 0, 0, 2684, 0, 0, 594, 0, 0, 426, 0, 0, 6, 0, 0, 0, 4092, 0, 324, 0, 2370, 2609, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 4076] payload_rows_per_dst=[8168, 5152, 7250, 7562, 8192, 648, 8152, 6960] dup_rows_per_dst=[5886, 8, 1968, 3734, 856, 0, 1000, 0] combine_rows_per_src=[8168, 5152, 7250, 7562, 8192, 648, 8152, 6960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=193634304 dispatch_meta_bytes=219248 combine_payload_bytes=193634304 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=193853552 ingress_bytes=218800128 link_bytes=218800128 token_bytes=387487856 weight_movement_bytes=25165824 total_nvlink_bytes=412653680 weight_to_token_ratio=0.064946 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.664816165e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54020 payload_rows_remote=45830 dup_rows=11516 dup_rows_remote=11472 dedup_ratio=0.175720 router_tokens_per_expert=[0, 0, 0, 3621, 0, 764, 0, 0, 0, 0, 0, 173, 0, 1116, 0, 0, 0, 3436, 16, 1332, 0, 0, 0, 0, 4051, 4004, 4078, 0, 0, 3762, 0, 0, 248, 0, 0, 0, 0, 942, 10, 0, 0, 0, 0, 0, 1, 4093, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 563, 0, 0, 535, 0, 0, 0, 0] payload_rows_per_dst=[7424, 6002, 7298, 5934, 7922, 8190, 8172, 3078] dup_rows_per_dst=[1346, 1368, 2270, 0, 2002, 44, 3526, 960] combine_rows_per_src=[7424, 6002, 7298, 5934, 7922, 8190, 8172, 3078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54810 payload_rows_remote=46800 dup_rows=10726 dup_rows_remote=8958 dedup_ratio=0.163666 router_tokens_per_expert=[1, 0, 0, 150, 0, 1989, 15, 0, 0, 0, 0, 22, 0, 2754, 0, 0, 0, 2577, 33, 57, 0, 0, 11, 3, 2789, 4075, 4074, 0, 0, 3987, 0, 0, 122, 0, 0, 0, 0, 615, 165, 0, 4, 1, 0, 0, 7, 4096, 0, 449, 0, 0, 226, 0, 0, 0, 0, 0, 1341, 1, 0, 3204, 0, 0, 0, 0] payload_rows_per_dst=[4214, 8170, 5206, 5578, 8010, 8192, 8182, 7258] dup_rows_per_dst=[96, 5532, 156, 0, 1768, 922, 418, 1834] combine_rows_per_src=[4214, 8170, 5206, 5578, 8010, 8192, 8182, 7258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54978 payload_rows_remote=46862 dup_rows=10558 dup_rows_remote=8254 dedup_ratio=0.161102 router_tokens_per_expert=[13, 0, 0, 2186, 0, 2957, 0, 0, 0, 0, 6, 22, 0, 1254, 4, 0, 5, 3302, 8, 257, 1, 0, 67, 0, 3199, 4056, 2376, 0, 2, 3292, 0, 0, 2649, 8, 0, 0, 0, 223, 131, 0, 0, 0, 0, 0, 243, 3779, 0, 0, 0, 3, 0, 0, 0, 0, 4, 3, 2009, 0, 0, 705, 0, 2, 2, 0] payload_rows_per_dst=[7434, 8148, 6744, 8116, 6864, 7962, 4762, 4948] dup_rows_per_dst=[2878, 2536, 536, 2304, 1724, 82, 10, 488] combine_rows_per_src=[7434, 8148, 6744, 8116, 6864, 7962, 4762, 4948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57008 payload_rows_remote=48826 dup_rows=8528 dup_rows_remote=7732 dedup_ratio=0.130127 router_tokens_per_expert=[0, 0, 0, 147, 0, 690, 13, 0, 4, 0, 0, 387, 2, 6, 0, 0, 0, 3575, 218, 1450, 0, 0, 6, 0, 328, 4090, 3924, 0, 0, 4060, 0, 0, 2377, 0, 0, 0, 0, 2177, 35, 0, 0, 0, 0, 0, 121, 4083, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, 4082, 0, 0, 0, 0] payload_rows_per_dst=[1656, 8182, 7776, 8144, 7054, 8184, 7848, 8164] dup_rows_per_dst=[44, 796, 2722, 632, 2124, 1658, 0, 552] combine_rows_per_src=[1656, 8182, 7776, 8144, 7054, 8184, 7848, 8164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=179879936 dispatch_meta_bytes=205928 combine_payload_bytes=179879936 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=180085864 ingress_bytes=255377408 link_bytes=255377408 token_bytes=359965800 weight_movement_bytes=75497472 total_nvlink_bytes=435463272 weight_to_token_ratio=0.209735 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.943127003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55858 payload_rows_remote=48580 dup_rows=9678 dup_rows_remote=7228 dedup_ratio=0.147675 router_tokens_per_expert=[0, 0, 0, 1150, 0, 3049, 6, 7, 0, 0, 0, 47, 0, 775, 0, 1, 0, 3851, 50, 725, 0, 6, 0, 24, 1182, 3724, 4082, 0, 0, 4085, 0, 0, 451, 0, 0, 0, 5, 1690, 6, 11, 0, 0, 0, 0, 137, 4012, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2793, 0, 2, 887, 0, 0, 0, 0] payload_rows_per_dst=[7152, 1638, 7862, 7448, 8178, 8138, 8164, 7278] dup_rows_per_dst=[1272, 8, 1450, 0, 4318, 180, 0, 2450] combine_rows_per_src=[7152, 1638, 7862, 7448, 8178, 8138, 8164, 7278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57282 payload_rows_remote=49090 dup_rows=8254 dup_rows_remote=8254 dedup_ratio=0.125946 router_tokens_per_expert=[0, 0, 0, 3495, 0, 379, 1, 0, 0, 0, 0, 116, 0, 1653, 0, 0, 0, 3978, 8, 1172, 0, 0, 24, 0, 2260, 3825, 4096, 0, 0, 3732, 0, 0, 2029, 0, 0, 0, 1, 15, 15, 0, 0, 0, 0, 17, 0, 4096, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1780, 0, 0, 0, 0] payload_rows_per_dst=[7178, 3524, 8110, 7650, 7946, 8192, 8192, 6490] dup_rows_per_dst=[572, 14, 2254, 0, 3638, 186, 0, 1590] combine_rows_per_src=[7178, 3524, 8110, 7650, 7946, 8192, 8192, 6490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=199991296 dispatch_meta_bytes=226232 combine_payload_bytes=199991296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200217528 ingress_bytes=225157120 link_bytes=225157120 token_bytes=400208824 weight_movement_bytes=25165824 total_nvlink_bytes=425374648 weight_to_token_ratio=0.062882 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20992 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.713185529e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=187719680 dispatch_meta_bytes=229208 combine_payload_bytes=187719680 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=187948888 ingress_bytes=238051328 link_bytes=238051328 token_bytes=375668568 weight_movement_bytes=50331648 total_nvlink_bytes=426000216 weight_to_token_ratio=0.133979 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.811295553e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=191692800 dispatch_meta_bytes=223032 combine_payload_bytes=191692800 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=191915832 ingress_bytes=216858624 link_bytes=216858624 token_bytes=383608632 weight_movement_bytes=25165824 total_nvlink_bytes=408774456 weight_to_token_ratio=0.065603 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16896 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.650043562e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=191946752 dispatch_meta_bytes=220464 combine_payload_bytes=191946752 prefetch_bytes=25165824 reduce_bytes=125829120 egress_bytes=192167216 ingress_bytes=342941696 link_bytes=342941696 token_bytes=384113968 weight_movement_bytes=150994944 total_nvlink_bytes=535108912 weight_to_token_ratio=0.393099 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=11776 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.609390059e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=198983680 dispatch_meta_bytes=223232 combine_payload_bytes=198983680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=199206912 ingress_bytes=224149504 link_bytes=224149504 token_bytes=398190592 weight_movement_bytes=25165824 total_nvlink_bytes=423356416 weight_to_token_ratio=0.063200 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=18560 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.705518735e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53278 payload_rows_remote=45126 dup_rows=12258 dup_rows_remote=9666 dedup_ratio=0.187042 router_tokens_per_expert=[0, 0, 0, 4075, 1, 3154, 5, 0, 24, 0, 0, 131, 0, 1498, 0, 0, 0, 4018, 130, 1224, 0, 0, 0, 0, 1448, 2278, 4066, 0, 0, 1675, 0, 0, 625, 0, 0, 0, 0, 0, 11, 0, 0, 11, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 3775, 0, 0, 0, 0] payload_rows_per_dst=[8190, 6160, 8152, 5556, 1272, 8192, 8132, 7624] dup_rows_per_dst=[6280, 1702, 2592, 690, 0, 56, 0, 938] combine_rows_per_src=[8190, 6160, 8152, 5556, 1272, 8192, 8132, 7624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=201072640 dispatch_meta_bytes=229376 combine_payload_bytes=201072640 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=201302016 ingress_bytes=251404288 link_bytes=251404288 token_bytes=402374656 weight_movement_bytes=50331648 total_nvlink_bytes=452706304 weight_to_token_ratio=0.125087 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.912896150e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50274 payload_rows_remote=42448 dup_rows=15262 dup_rows_remote=13288 dedup_ratio=0.232880 router_tokens_per_expert=[0, 0, 0, 3679, 0, 948, 0, 0, 3, 0, 0, 18, 0, 1793, 0, 0, 0, 1159, 138, 34, 0, 0, 0, 12, 1558, 4064, 4065, 0, 0, 3905, 0, 0, 145, 0, 0, 0, 0, 11, 1202, 0, 0, 0, 0, 0, 5, 4086, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2433, 0, 0, 3509, 0, 0, 0, 0] payload_rows_per_dst=[7826, 8166, 2176, 8094, 2694, 5356, 8130, 7832] dup_rows_per_dst=[1974, 3590, 232, 5390, 22, 2, 0, 4052] combine_rows_per_src=[7826, 8166, 2176, 8094, 2694, 5356, 8130, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=184836096 dispatch_meta_bytes=219168 combine_payload_bytes=184836096 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=185055264 ingress_bytes=235167744 link_bytes=235167744 token_bytes=369891360 weight_movement_bytes=50331648 total_nvlink_bytes=420223008 weight_to_token_ratio=0.136071 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789354811e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=173867008 dispatch_meta_bytes=222944 combine_payload_bytes=173867008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=174089952 ingress_bytes=199032832 link_bytes=199032832 token_bytes=347956960 weight_movement_bytes=25165824 total_nvlink_bytes=373122784 weight_to_token_ratio=0.072325 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18560 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.514409882e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53228 payload_rows_remote=45036 dup_rows=12308 dup_rows_remote=5190 dedup_ratio=0.187805 router_tokens_per_expert=[3, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3498, 4062, 0, 94, 5, 0, 0, 0, 0, 1078, 0, 1353, 0, 0, 360, 4088, 2, 3840, 0, 950, 669, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 4028, 0, 0, 0, 449, 317, 5, 0, 1, 0, 0, 2212, 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0] payload_rows_per_dst=[8190, 8192, 4756, 8054, 8192, 8146, 4434, 3264] dup_rows_per_dst=[26, 7118, 116, 2250, 1338, 1458, 2, 0] combine_rows_per_src=[8190, 8192, 4756, 8054, 8192, 8146, 4434, 3264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52678 payload_rows_remote=49024 dup_rows=12858 dup_rows_remote=12858 dedup_ratio=0.196198 router_tokens_per_expert=[1, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 1186, 4086, 0, 79, 0, 0, 0, 0, 0, 543, 0, 1310, 0, 0, 0, 4096, 0, 3053, 0, 2839, 45, 0, 0, 0, 4095, 0, 9, 0, 0, 0, 2210, 0, 1, 0, 1826, 0, 0, 0, 0, 0, 0, 3595, 1, 0, 0, 0, 0, 0, 0, 3717, 0, 0] payload_rows_per_dst=[8192, 2476, 8182, 6106, 8192, 3654, 7914, 7962] dup_rows_per_dst=[154, 54, 3696, 0, 106, 0, 4956, 3892] combine_rows_per_src=[8192, 2476, 8182, 6106, 8192, 3654, 7914, 7962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53328 payload_rows_remote=46668 dup_rows=12208 dup_rows_remote=9944 dedup_ratio=0.186279 router_tokens_per_expert=[112, 0, 0, 0, 0, 97, 0, 0, 8, 0, 5, 1, 1197, 3934, 0, 218, 0, 0, 0, 0, 0, 3686, 2, 9, 0, 0, 0, 4086, 0, 1743, 0, 2719, 936, 1, 9, 0, 4089, 10, 18, 0, 0, 0, 3550, 0, 21, 3, 2753, 19, 0, 5, 6, 0, 0, 1859, 56, 3, 0, 14, 5, 0, 0, 1587, 7, 0] payload_rows_per_dst=[8174, 8140, 7394, 6660, 8192, 7474, 3852, 3442] dup_rows_per_dst=[416, 2586, 0, 2264, 1934, 3948, 6, 1054] combine_rows_per_src=[8174, 8140, 7394, 6660, 8192, 7474, 3852, 3442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51832 payload_rows_remote=43640 dup_rows=13704 dup_rows_remote=13302 dedup_ratio=0.209106 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2056, 3091, 0, 0, 10, 0, 0, 0, 0, 167, 0, 480, 0, 0, 139, 4090, 0, 2981, 0, 3314, 202, 0, 0, 0, 4092, 0, 0, 3, 0, 0, 3940, 0, 1, 0, 2139, 279, 0, 0, 0, 0, 0, 1601, 132, 0, 0, 0, 0, 0, 0, 4046, 0, 0] payload_rows_per_dst=[8190, 4136, 6716, 7826, 8192, 4714, 3966, 8092] dup_rows_per_dst=[0, 172, 584, 4272, 402, 124, 270, 7880] combine_rows_per_src=[8190, 4136, 6716, 7826, 8192, 4714, 3966, 8092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47768 payload_rows_remote=41110 dup_rows=17768 dup_rows_remote=14482 dedup_ratio=0.271118 router_tokens_per_expert=[0, 0, 39, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1911, 3937, 0, 127, 0, 0, 0, 0, 0, 16, 0, 2727, 0, 0, 973, 4093, 0, 3038, 1, 2336, 192, 0, 0, 0, 4086, 0, 11, 4, 0, 19, 1689, 0, 0, 0, 396, 7, 3, 0, 2, 1, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 3283, 0, 0] payload_rows_per_dst=[6478, 254, 8150, 7954, 3074, 7098, 8102, 6658] dup_rows_per_dst=[80, 0, 5210, 3924, 148, 792, 4328, 3286] combine_rows_per_src=[6478, 254, 8150, 7954, 3074, 7098, 8102, 6658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51624 payload_rows_remote=43520 dup_rows=13912 dup_rows_remote=8106 dedup_ratio=0.212280 router_tokens_per_expert=[0, 0, 15, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3475, 4096, 0, 2, 0, 0, 0, 0, 0, 1818, 0, 74, 0, 0, 140, 4096, 0, 1884, 0, 3266, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 2868, 0, 0, 0, 925, 34, 0, 0, 3, 0, 0, 3618, 68, 0, 0, 0, 0, 0, 0, 2286, 0, 0] payload_rows_per_dst=[8192, 6698, 8192, 3914, 8186, 1918, 8104, 6420] dup_rows_per_dst=[38, 4, 3784, 386, 6, 0, 5806, 3888] combine_rows_per_src=[8192, 6698, 8192, 3914, 8186, 1918, 8104, 6420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=184467456 dispatch_meta_bytes=200904 combine_payload_bytes=184467456 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=184668360 ingress_bytes=259964928 link_bytes=259964928 token_bytes=369135816 weight_movement_bytes=75497472 total_nvlink_bytes=444633288 weight_to_token_ratio=0.204525 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=16256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978032729e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=178749440 dispatch_meta_bytes=227768 combine_payload_bytes=178749440 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=178977208 ingress_bytes=229081088 link_bytes=229081088 token_bytes=357726648 weight_movement_bytes=50331648 total_nvlink_bytes=408058296 weight_to_token_ratio=0.140699 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29312 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.743042391e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=200802304 dispatch_meta_bytes=247528 combine_payload_bytes=200802304 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=201049832 ingress_bytes=251133952 link_bytes=251133952 token_bytes=401852136 weight_movement_bytes=50331648 total_nvlink_bytes=452183784 weight_to_token_ratio=0.125249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=14336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.910839205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=191152128 dispatch_meta_bytes=226448 combine_payload_bytes=191152128 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=191378576 ingress_bytes=266649600 link_bytes=266649600 token_bytes=382530704 weight_movement_bytes=75497472 total_nvlink_bytes=458028176 weight_to_token_ratio=0.197363 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18048 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.028895359e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=168386560 dispatch_meta_bytes=222368 combine_payload_bytes=168386560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=168608928 ingress_bytes=193552384 link_bytes=193552384 token_bytes=336995488 weight_movement_bytes=25165824 total_nvlink_bytes=362161312 weight_to_token_ratio=0.074677 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=21504 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.472710005e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49596 payload_rows_remote=47456 dup_rows=15940 dup_rows_remote=15890 dedup_ratio=0.243225 router_tokens_per_expert=[0, 0, 5, 0, 0, 451, 1, 0, 0, 0, 0, 0, 3765, 4095, 0, 0, 0, 0, 0, 0, 0, 569, 0, 526, 0, 0, 29, 4094, 0, 824, 0, 1578, 84, 1, 0, 0, 4096, 0, 0, 5, 0, 2, 3687, 0, 0, 0, 3768, 53, 0, 0, 3, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 0] payload_rows_per_dst=[8192, 8192, 2140, 4454, 8192, 8156, 7058, 3212] dup_rows_per_dst=[910, 7528, 50, 408, 180, 6864, 0, 0] combine_rows_per_src=[8192, 8192, 2140, 4454, 8192, 8156, 7058, 3212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58680 payload_rows_remote=50498 dup_rows=6856 dup_rows_remote=6734 dedup_ratio=0.104614 router_tokens_per_expert=[0, 0, 62, 0, 0, 1, 0, 0, 0, 0, 0, 0, 967, 3996, 0, 30, 10, 0, 0, 0, 0, 3235, 0, 38, 0, 0, 4, 4089, 0, 597, 0, 2931, 505, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3660, 0, 0, 0, 1927, 8, 0, 0, 0, 0, 0, 3345, 24, 0, 2, 0, 0, 0, 0, 3241, 0, 0] payload_rows_per_dst=[8182, 8150, 6542, 6452, 8192, 7960, 6716, 6486] dup_rows_per_dst=[122, 1836, 24, 612, 1010, 3230, 22, 0] combine_rows_per_src=[8182, 8150, 6542, 6452, 8192, 7960, 6716, 6486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=178257920 dispatch_meta_bytes=206504 combine_payload_bytes=178257920 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=178464424 ingress_bytes=203423744 link_bytes=203423744 token_bytes=356722344 weight_movement_bytes=25165824 total_nvlink_bytes=381888168 weight_to_token_ratio=0.070547 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.547819649e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=194379776 dispatch_meta_bytes=253384 combine_payload_bytes=194379776 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=194633160 ingress_bytes=219545600 link_bytes=219545600 token_bytes=389012936 weight_movement_bytes=25165824 total_nvlink_bytes=414178760 weight_to_token_ratio=0.064691 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.670488345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=206839808 dispatch_meta_bytes=228928 combine_payload_bytes=206839808 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=207068736 ingress_bytes=232005632 link_bytes=232005632 token_bytes=413908544 weight_movement_bytes=25165824 total_nvlink_bytes=439074368 weight_to_token_ratio=0.060800 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.765294792e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52846 payload_rows_remote=44656 dup_rows=12690 dup_rows_remote=12664 dedup_ratio=0.193634 router_tokens_per_expert=[0, 0, 0, 2065, 0, 0, 4075, 368, 19, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 16, 0, 1202, 4076, 11, 0, 15, 110, 334, 0, 0, 0, 0, 0, 0, 1, 467, 1663, 5, 0, 1, 4, 3705, 0, 0, 0, 2511, 4087, 0, 3655, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 1, 1, 14, 97] payload_rows_per_dst=[8186, 8190, 8174, 8192, 4262, 8184, 7436, 222] dup_rows_per_dst=[4830, 26, 2414, 922, 10, 4258, 226, 4] combine_rows_per_src=[8186, 8190, 8174, 8192, 4262, 8184, 7436, 222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51994 payload_rows_remote=44738 dup_rows=13542 dup_rows_remote=13518 dedup_ratio=0.206635 router_tokens_per_expert=[0, 0, 0, 1057, 0, 0, 4065, 7, 0, 0, 0, 0, 0, 0, 3586, 0, 0, 0, 0, 0, 0, 0, 1848, 4086, 0, 0, 0, 114, 73, 0, 0, 0, 0, 0, 0, 2, 50, 3578, 9, 1, 0, 0, 4037, 0, 0, 0, 599, 4034, 0, 3792, 0, 0, 11, 0, 0, 1, 0, 0, 16, 0, 0, 0, 1726, 76] payload_rows_per_dst=[8144, 7172, 3696, 8170, 7256, 1198, 8168, 8190] dup_rows_per_dst=[2114, 0, 0, 272, 24, 0, 7514, 3618] combine_rows_per_src=[8144, 7172, 3696, 8170, 7256, 1198, 8168, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52728 payload_rows_remote=44542 dup_rows=12808 dup_rows_remote=10958 dedup_ratio=0.195435 router_tokens_per_expert=[2, 0, 3, 63, 0, 0, 3599, 45, 0, 0, 21, 0, 2, 11, 4046, 0, 7, 2, 17, 0, 754, 0, 1446, 4090, 0, 5, 211, 688, 160, 5, 6, 5, 0, 0, 0, 648, 98, 4024, 25, 24, 17, 0, 3945, 0, 0, 1, 339, 3938, 0, 980, 0, 0, 0, 1, 0, 6, 0, 0, 942, 17, 0, 0, 2575, 0] payload_rows_per_dst=[7292, 8114, 4278, 8186, 8122, 4828, 3716, 8192] dup_rows_per_dst=[132, 46, 174, 1850, 1516, 426, 1608, 7056] combine_rows_per_src=[7292, 8114, 4278, 8186, 8122, 4828, 3716, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56704 payload_rows_remote=51618 dup_rows=8832 dup_rows_remote=8832 dedup_ratio=0.134766 router_tokens_per_expert=[0, 0, 0, 28, 1, 0, 2877, 206, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 2, 0, 3322, 3988, 0, 0, 24, 380, 45, 0, 1, 0, 0, 0, 0, 361, 712, 4070, 94, 0, 0, 0, 2037, 0, 0, 0, 2543, 4095, 0, 1826, 0, 0, 18, 0, 3, 3, 0, 0, 138, 0, 0, 0, 1998, 7] payload_rows_per_dst=[6210, 7978, 6648, 8192, 8152, 5086, 6390, 8048] dup_rows_per_dst=[14, 0, 0, 898, 2322, 0, 1384, 4214] combine_rows_per_src=[6210, 7978, 6648, 8192, 8152, 5086, 6390, 8048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50966 payload_rows_remote=42848 dup_rows=14570 dup_rows_remote=14454 dedup_ratio=0.222321 router_tokens_per_expert=[0, 0, 0, 2689, 0, 0, 3875, 6, 0, 0, 6, 1, 0, 0, 3915, 0, 0, 0, 0, 0, 93, 0, 1699, 4048, 0, 0, 11, 12, 556, 0, 0, 0, 0, 0, 0, 176, 67, 2776, 0, 0, 0, 5, 3420, 0, 0, 0, 2268, 4078, 0, 2910, 0, 0, 0, 0, 88, 0, 0, 0, 26, 0, 3, 0, 40, 0] payload_rows_per_dst=[5378, 7842, 7966, 1118, 4608, 8174, 7762, 8118] dup_rows_per_dst=[12, 2, 3368, 40, 98, 5860, 5074, 116] combine_rows_per_src=[5378, 7842, 7966, 1118, 4608, 8174, 7762, 8118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53564 payload_rows_remote=45982 dup_rows=11972 dup_rows_remote=9862 dedup_ratio=0.182678 router_tokens_per_expert=[0, 0, 0, 39, 0, 0, 3146, 306, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 905, 3953, 0, 0, 0, 547, 275, 0, 0, 0, 0, 0, 0, 629, 95, 4066, 2, 0, 0, 0, 3736, 0, 0, 0, 3074, 4081, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 2533, 38] payload_rows_per_dst=[2922, 8192, 4662, 7636, 8154, 6314, 7582, 8102] dup_rows_per_dst=[74, 0, 1134, 1610, 1430, 394, 2110, 5220] combine_rows_per_src=[2922, 8192, 4662, 7636, 8154, 6314, 7582, 8102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=182910976 dispatch_meta_bytes=229280 combine_payload_bytes=182910976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183140256 ingress_bytes=208076800 link_bytes=208076800 token_bytes=366051232 weight_movement_bytes=25165824 total_nvlink_bytes=391217056 weight_to_token_ratio=0.068749 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.583224029e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=183246848 dispatch_meta_bytes=233024 combine_payload_bytes=183246848 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=183479872 ingress_bytes=258744320 link_bytes=258744320 token_bytes=366726720 weight_movement_bytes=75497472 total_nvlink_bytes=442224192 weight_to_token_ratio=0.205868 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=25216 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.968745313e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=182444032 dispatch_meta_bytes=222000 combine_payload_bytes=182444032 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=182666032 ingress_bytes=207609856 link_bytes=207609856 token_bytes=365110064 weight_movement_bytes=25165824 total_nvlink_bytes=390275888 weight_to_token_ratio=0.068927 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.579671124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=211427328 dispatch_meta_bytes=241800 combine_payload_bytes=211427328 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=211669128 ingress_bytes=286924800 link_bytes=286924800 token_bytes=423096456 weight_movement_bytes=75497472 total_nvlink_bytes=498593928 weight_to_token_ratio=0.178440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=13952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.183166205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=175505408 dispatch_meta_bytes=229208 combine_payload_bytes=175505408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=175734616 ingress_bytes=200671232 link_bytes=200671232 token_bytes=351240024 weight_movement_bytes=25165824 total_nvlink_bytes=376405848 weight_to_token_ratio=0.071649 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.526876213e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50732 payload_rows_remote=43684 dup_rows=14804 dup_rows_remote=10368 dedup_ratio=0.225891 router_tokens_per_expert=[0, 0, 0, 321, 0, 0, 2768, 443, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 16, 0, 2852, 3903, 0, 0, 1, 84, 108, 0, 0, 0, 0, 0, 0, 338, 2563, 3975, 170, 0, 0, 0, 3919, 0, 0, 0, 1616, 4095, 0, 1277, 0, 0, 3, 0, 3, 0, 0, 0, 48, 0, 0, 1, 315, 3] payload_rows_per_dst=[6466, 7892, 7048, 8190, 8128, 8162, 2564, 2282] dup_rows_per_dst=[598, 0, 4436, 386, 5964, 2908, 2, 510] combine_rows_per_src=[6466, 7892, 7048, 8190, 8128, 8162, 2564, 2282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50712 payload_rows_remote=42528 dup_rows=14824 dup_rows_remote=9844 dedup_ratio=0.226196 router_tokens_per_expert=[0, 0, 0, 1587, 0, 0, 3627, 197, 0, 0, 47, 0, 0, 0, 3826, 0, 0, 0, 0, 0, 9, 0, 1807, 3953, 0, 0, 1, 449, 440, 0, 1, 0, 0, 0, 0, 1112, 1, 3971, 0, 0, 0, 33, 4039, 0, 0, 0, 800, 4046, 0, 1935, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 786, 36] payload_rows_per_dst=[8184, 7738, 8086, 8192, 4722, 8164, 3982, 1644] dup_rows_per_dst=[4980, 2344, 3452, 1682, 768, 1580, 18, 0] combine_rows_per_src=[8184, 7738, 8086, 8192, 4722, 8164, 3982, 1644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=188342272 dispatch_meta_bytes=223376 combine_payload_bytes=188342272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=188565648 ingress_bytes=213508096 link_bytes=213508096 token_bytes=376907920 weight_movement_bytes=25165824 total_nvlink_bytes=402073744 weight_to_token_ratio=0.066769 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=17536 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.624549916e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=178929664 dispatch_meta_bytes=216208 combine_payload_bytes=178929664 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=179145872 ingress_bytes=229261312 link_bytes=229261312 token_bytes=358075536 weight_movement_bytes=50331648 total_nvlink_bytes=408407184 weight_to_token_ratio=0.140562 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=15104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.744413688e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=174194688 dispatch_meta_bytes=209488 combine_payload_bytes=174194688 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=174404176 ingress_bytes=224526336 link_bytes=224526336 token_bytes=348598864 weight_movement_bytes=50331648 total_nvlink_bytes=398930512 weight_to_token_ratio=0.144383 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.708385991e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54754 payload_rows_remote=46894 dup_rows=10782 dup_rows_remote=5812 dedup_ratio=0.164520 router_tokens_per_expert=[0, 4078, 0, 547, 0, 0, 4084, 7, 3393, 3877, 0, 0, 748, 0, 0, 0, 126, 0, 0, 4069, 0, 1, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 1048, 0, 0, 4, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1114, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8160, 7860, 8192, 6786, 5292, 8068, 2228, 8168] dup_rows_per_dst=[1104, 4970, 3496, 0, 1210, 0, 2, 0] combine_rows_per_src=[8160, 7860, 8192, 6786, 5292, 8068, 2228, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53292 payload_rows_remote=45124 dup_rows=12244 dup_rows_remote=11936 dedup_ratio=0.186829 router_tokens_per_expert=[0, 4086, 10, 2324, 37, 6, 4087, 0, 4009, 1629, 0, 0, 477, 0, 10, 0, 887, 3, 0, 4071, 0, 0, 0, 3146, 0, 0, 61, 0, 148, 12, 0, 8, 8, 0, 0, 3963, 346, 14, 8, 0, 0, 0, 0, 738, 7, 0, 0, 0, 1, 7, 0, 6, 55, 6, 0, 2588, 1, 0, 3, 0, 0, 0, 6, 0] payload_rows_per_dst=[8192, 4084, 7016, 8168, 7980, 1476, 8184, 8192] dup_rows_per_dst=[4734, 148, 1056, 308, 698, 14, 5284, 2] combine_rows_per_src=[8192, 4084, 7016, 8168, 7980, 1476, 8184, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54110 payload_rows_remote=46026 dup_rows=11426 dup_rows_remote=10094 dedup_ratio=0.174347 router_tokens_per_expert=[0, 3829, 0, 331, 0, 0, 4086, 14, 3926, 3667, 0, 0, 2806, 0, 0, 0, 278, 0, 0, 4093, 0, 0, 0, 626, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 668, 0, 0, 0, 0, 0, 0, 2423, 1, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 1924, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6844, 8050, 1774, 7858, 8084, 5136, 8192, 8172] dup_rows_per_dst=[538, 4896, 34, 0, 1332, 678, 3948, 0] combine_rows_per_src=[6844, 8050, 1774, 7858, 8084, 5136, 8192, 8172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58650 payload_rows_remote=50458 dup_rows=6886 dup_rows_remote=3768 dedup_ratio=0.105072 router_tokens_per_expert=[0, 4096, 0, 3057, 0, 0, 4096, 0, 3990, 3718, 0, 0, 4, 0, 0, 0, 1789, 0, 0, 4096, 0, 0, 0, 797, 0, 0, 103, 0, 21, 0, 0, 0, 0, 0, 2, 4079, 588, 0, 0, 0, 0, 0, 0, 1555, 4, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6114, 7442, 4242, 8118, 8158, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 930, 110, 1180, 3118, 1546, 0] combine_rows_per_src=[6114, 7442, 4242, 8118, 8158, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47316 payload_rows_remote=39146 dup_rows=18220 dup_rows_remote=18140 dedup_ratio=0.278015 router_tokens_per_expert=[0, 3530, 0, 150, 0, 0, 4085, 0, 4045, 4008, 0, 0, 421, 0, 0, 4, 365, 0, 0, 4096, 0, 0, 0, 1541, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 3920, 1938, 0, 0, 0, 2, 0, 0, 1947, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[300, 5694, 1204, 8106, 7890, 7760, 8192, 8170] dup_rows_per_dst=[0, 520, 122, 2648, 6314, 3126, 5410, 80] combine_rows_per_src=[300, 5694, 1204, 8106, 7890, 7760, 8192, 8170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=192077824 dispatch_meta_bytes=210824 combine_payload_bytes=192077824 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=192288648 ingress_bytes=267575296 link_bytes=267575296 token_bytes=384366472 weight_movement_bytes=75497472 total_nvlink_bytes=459863944 weight_to_token_ratio=0.196421 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=22272 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.035938836e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55998 payload_rows_remote=47810 dup_rows=9538 dup_rows_remote=9362 dedup_ratio=0.145538 router_tokens_per_expert=[0, 4096, 0, 877, 0, 0, 4096, 11, 3869, 3448, 0, 0, 914, 0, 0, 0, 1944, 0, 0, 4094, 0, 0, 0, 2540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 196, 0, 0, 0, 0, 0, 0, 2594, 3, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1776, 7310, 6606, 7738, 7996, 8192, 8188, 8192] dup_rows_per_dst=[0, 1414, 2362, 0, 392, 5194, 176, 0] combine_rows_per_src=[1776, 7310, 6606, 7738, 7996, 8192, 8188, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=184827904 dispatch_meta_bytes=228240 combine_payload_bytes=184827904 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=185056144 ingress_bytes=235159552 link_bytes=235159552 token_bytes=369884048 weight_movement_bytes=50331648 total_nvlink_bytes=420215696 weight_to_token_ratio=0.136074 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=30976 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789292479e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=188522496 dispatch_meta_bytes=224480 combine_payload_bytes=188522496 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=188746976 ingress_bytes=264019968 link_bytes=264019968 token_bytes=377269472 weight_movement_bytes=75497472 total_nvlink_bytes=452766944 weight_to_token_ratio=0.200116 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.008886898e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=206675968 dispatch_meta_bytes=216904 combine_payload_bytes=206675968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206892872 ingress_bytes=231841792 link_bytes=231841792 token_bytes=413568840 weight_movement_bytes=25165824 total_nvlink_bytes=438734664 weight_to_token_ratio=0.060850 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.764048159e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160342016 dispatch_meta_bytes=229144 combine_payload_bytes=160342016 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=160571160 ingress_bytes=210673664 link_bytes=210673664 token_bytes=320913176 weight_movement_bytes=50331648 total_nvlink_bytes=371244824 weight_to_token_ratio=0.156839 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602983163e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56364 payload_rows_remote=48780 dup_rows=9172 dup_rows_remote=2612 dedup_ratio=0.139954 router_tokens_per_expert=[0, 4088, 0, 370, 0, 0, 4088, 16, 3800, 2474, 0, 0, 523, 0, 0, 0, 529, 0, 0, 4095, 0, 0, 0, 3367, 0, 0, 81, 0, 33, 0, 0, 0, 0, 0, 0, 4095, 341, 0, 0, 28, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 1, 1079, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5548, 7584, 7782, 8192, 7506, 3384, 8176] dup_rows_per_dst=[756, 446, 6560, 46, 736, 0, 628, 0] combine_rows_per_src=[8192, 5548, 7584, 7782, 8192, 7506, 3384, 8176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=46962 payload_rows_remote=38770 dup_rows=18574 dup_rows_remote=7940 dedup_ratio=0.283417 router_tokens_per_expert=[0, 4071, 0, 2640, 0, 0, 4096, 80, 3918, 749, 0, 0, 1353, 0, 0, 0, 301, 0, 0, 4096, 0, 0, 0, 2782, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 0, 0, 0, 0, 0, 3537, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5380, 8192, 7866, 82, 7076, 1982, 8192] dup_rows_per_dst=[10634, 1772, 6166, 2, 0, 0, 0, 0] combine_rows_per_src=[8192, 5380, 8192, 7866, 82, 7076, 1982, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=195829760 dispatch_meta_bytes=228688 combine_payload_bytes=195829760 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=196058448 ingress_bytes=220995584 link_bytes=220995584 token_bytes=391888208 weight_movement_bytes=25165824 total_nvlink_bytes=417054032 weight_to_token_ratio=0.064217 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.681521048e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57688 payload_rows_remote=49528 dup_rows=7848 dup_rows_remote=2464 dedup_ratio=0.119751 router_tokens_per_expert=[0, 0, 0, 0, 0, 270, 345, 0, 0, 2, 2653, 0, 33, 32, 0, 0, 0, 3558, 0, 423, 2, 0, 0, 0, 4091, 0, 0, 26, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 283, 4011, 0, 4096, 0, 0, 0, 0, 4052, 0, 4069, 0, 0, 0, 0, 609, 4094, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8160, 7578, 8192, 8138, 8022, 1218, 8188] dup_rows_per_dst=[1230, 5384, 388, 42, 238, 566, 0, 0] combine_rows_per_src=[8192, 8160, 7578, 8192, 8138, 8022, 1218, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=199802880 dispatch_meta_bytes=205568 combine_payload_bytes=199802880 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=200008448 ingress_bytes=250134528 link_bytes=250134528 token_bytes=399811328 weight_movement_bytes=50331648 total_nvlink_bytes=450142976 weight_to_token_ratio=0.125888 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.903234744e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158801920 dispatch_meta_bytes=186840 combine_payload_bytes=158801920 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=158988760 ingress_bytes=259465216 link_bytes=259465216 token_bytes=317790680 weight_movement_bytes=100663296 total_nvlink_bytes=418453976 weight_to_token_ratio=0.316760 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.974230498e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62232 payload_rows_remote=54056 dup_rows=3304 dup_rows_remote=2826 dedup_ratio=0.050415 router_tokens_per_expert=[0, 0, 12, 0, 6, 1, 0, 0, 0, 0, 3953, 45, 1, 14, 0, 0, 151, 3734, 6, 686, 0, 0, 0, 0, 4072, 14, 0, 234, 7, 0, 0, 0, 56, 68, 5, 0, 8, 0, 0, 0, 7, 3488, 0, 4038, 0, 7, 30, 0, 838, 0, 4071, 0, 7, 3, 0, 3520, 3662, 0, 2, 0, 2, 9, 7, 4] payload_rows_per_dst=[8102, 7936, 7674, 8176, 8178, 7050, 7774, 7342] dup_rows_per_dst=[12, 90, 1480, 478, 238, 14, 962, 30] combine_rows_per_src=[8102, 7936, 7674, 8176, 8178, 7050, 7774, 7342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62208 payload_rows_remote=54090 dup_rows=3328 dup_rows_remote=1942 dedup_ratio=0.050781 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 44, 0, 0, 0, 2881, 0, 28, 28, 0, 0, 127, 3700, 0, 24, 1, 0, 0, 0, 4093, 0, 0, 775, 0, 0, 0, 0, 695, 0, 0, 0, 0, 0, 0, 0, 5, 4024, 0, 4096, 0, 0, 0, 0, 4082, 0, 4057, 0, 0, 0, 0, 101, 3999, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5852, 7618, 8192, 8118, 8048, 8190, 7998] dup_rows_per_dst=[104, 22, 86, 1544, 1386, 10, 176, 0] combine_rows_per_src=[8192, 5852, 7618, 8192, 8118, 8048, 8190, 7998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60652 payload_rows_remote=52744 dup_rows=4884 dup_rows_remote=4884 dedup_ratio=0.074524 router_tokens_per_expert=[0, 0, 0, 0, 0, 22, 29, 0, 0, 0, 2951, 0, 14, 103, 0, 0, 16, 2864, 0, 140, 0, 0, 0, 0, 4096, 0, 0, 161, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 4096, 0, 0, 0, 0, 2872, 0, 4033, 0, 0, 0, 0, 3208, 4053, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6126, 5982, 8192, 8068, 7908, 8078, 8106] dup_rows_per_dst=[102, 10, 58, 322, 310, 0, 4082, 0] combine_rows_per_src=[8192, 6126, 5982, 8192, 8068, 7908, 8078, 8106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49764 payload_rows_remote=42762 dup_rows=15772 dup_rows_remote=10412 dedup_ratio=0.240662 router_tokens_per_expert=[0, 0, 0, 0, 0, 326, 105, 0, 0, 0, 3599, 0, 0, 0, 0, 0, 32, 3013, 0, 1252, 0, 0, 0, 7, 4052, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 233, 3917, 0, 4096, 4, 0, 0, 0, 3641, 0, 4095, 0, 0, 0, 0, 1107, 3287, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3368, 7198, 7598, 2318, 6348, 8102, 7830, 7002] dup_rows_per_dst=[860, 0, 1010, 0, 2, 4064, 4476, 5360] combine_rows_per_src=[3368, 7198, 7598, 2318, 6348, 8102, 7830, 7002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202866688 dispatch_meta_bytes=207968 combine_payload_bytes=202866688 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203074656 ingress_bytes=228032512 link_bytes=228032512 token_bytes=405941344 weight_movement_bytes=25165824 total_nvlink_bytes=431107168 weight_to_token_ratio=0.061994 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.735063939e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58342 payload_rows_remote=50180 dup_rows=7194 dup_rows_remote=2406 dedup_ratio=0.109772 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 1653, 0, 65, 36, 0, 0, 46, 2572, 0, 659, 0, 0, 0, 0, 3951, 0, 0, 1136, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 7, 3912, 0, 4096, 0, 0, 0, 0, 3873, 0, 4048, 0, 0, 0, 0, 2602, 4091, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 3370, 6406, 8096, 8096, 7838, 8162, 8182] dup_rows_per_dst=[22, 138, 148, 2078, 20, 0, 4788, 0] combine_rows_per_src=[8192, 3370, 6406, 8096, 8096, 7838, 8162, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=221413376 dispatch_meta_bytes=227528 combine_payload_bytes=221413376 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=221640904 ingress_bytes=296910848 link_bytes=296910848 token_bytes=443054280 weight_movement_bytes=75497472 total_nvlink_bytes=518551752 weight_to_token_ratio=0.170402 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.259148492e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=221552640 dispatch_meta_bytes=224128 combine_payload_bytes=221552640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=221776768 ingress_bytes=246718464 link_bytes=246718464 token_bytes=443329408 weight_movement_bytes=25165824 total_nvlink_bytes=468495232 weight_to_token_ratio=0.056766 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.877242444e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=216039424 dispatch_meta_bytes=230512 combine_payload_bytes=216039424 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=216269936 ingress_bytes=266371072 link_bytes=266371072 token_bytes=432309360 weight_movement_bytes=50331648 total_nvlink_bytes=482641008 weight_to_token_ratio=0.116425 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.026776083e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=175153152 dispatch_meta_bytes=212696 combine_payload_bytes=175153152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=175365848 ingress_bytes=200318976 link_bytes=200318976 token_bytes=350519000 weight_movement_bytes=25165824 total_nvlink_bytes=375684824 weight_to_token_ratio=0.071796 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.524195952e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=205537280 dispatch_meta_bytes=210344 combine_payload_bytes=205537280 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=205747624 ingress_bytes=306200576 link_bytes=306200576 token_bytes=411284904 weight_movement_bytes=100663296 total_nvlink_bytes=511948200 weight_to_token_ratio=0.244753 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=16128 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.329832588e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57202 payload_rows_remote=50354 dup_rows=8334 dup_rows_remote=8006 dedup_ratio=0.127167 router_tokens_per_expert=[0, 0, 0, 0, 0, 1143, 37, 0, 0, 0, 417, 0, 141, 1725, 0, 0, 12, 3388, 0, 188, 0, 0, 0, 0, 4069, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 0, 4094, 0, 0, 0, 0, 2863, 0, 4054, 0, 0, 0, 0, 3612, 3262, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4838, 6848, 8192, 8108, 6990, 7510, 6524] dup_rows_per_dst=[2356, 1608, 328, 2850, 0, 0, 1192, 0] combine_rows_per_src=[8192, 4838, 6848, 8192, 8108, 6990, 7510, 6524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53930 payload_rows_remote=45738 dup_rows=11606 dup_rows_remote=11378 dedup_ratio=0.177094 router_tokens_per_expert=[0, 0, 0, 0, 0, 96, 18, 0, 0, 0, 3926, 0, 165, 167, 0, 0, 240, 3380, 0, 551, 0, 0, 0, 0, 4084, 0, 0, 1344, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 0, 4096, 0, 0, 0, 0, 3144, 0, 4088, 0, 0, 0, 0, 549, 3402, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8154, 8180, 6402, 8176, 6924, 1098, 6804] dup_rows_per_dst=[228, 6650, 4154, 462, 112, 0, 0, 0] combine_rows_per_src=[8192, 8154, 8180, 6402, 8176, 6924, 1098, 6804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=50994 payload_rows_remote=42834 dup_rows=14542 dup_rows_remote=8796 dedup_ratio=0.221893 router_tokens_per_expert=[0, 11, 0, 7, 0, 4056, 301, 0, 2904, 0, 0, 4049, 0, 0, 0, 0, 0, 13, 0, 2621, 0, 12, 0, 0, 219, 45, 42, 0, 0, 0, 0, 170, 0, 0, 26, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4006, 0, 0, 0, 4092, 0, 2302, 4054, 0, 0, 6, 0, 0, 77, 0, 1, 3730, 0, 0, 0] payload_rows_per_dst=[8192, 8160, 5292, 938, 8192, 8012, 4614, 7594] dup_rows_per_dst=[8666, 5746, 0, 14, 92, 0, 2, 22] combine_rows_per_src=[8192, 8160, 5292, 938, 8192, 8012, 4614, 7594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=206249984 dispatch_meta_bytes=233440 combine_payload_bytes=206249984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206483424 ingress_bytes=231415808 link_bytes=231415808 token_bytes=412733408 weight_movement_bytes=25165824 total_nvlink_bytes=437899232 weight_to_token_ratio=0.060974 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=26240 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.760806913e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=187342848 dispatch_meta_bytes=228464 combine_payload_bytes=187342848 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=187571312 ingress_bytes=212508672 link_bytes=212508672 token_bytes=374914160 weight_movement_bytes=25165824 total_nvlink_bytes=400079984 weight_to_token_ratio=0.067124 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30336 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.616945454e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54878 payload_rows_remote=46704 dup_rows=10658 dup_rows_remote=7520 dedup_ratio=0.162628 router_tokens_per_expert=[0, 154, 2, 18, 0, 2381, 6, 9, 3798, 0, 0, 4007, 8, 0, 0, 10, 0, 0, 0, 3908, 0, 0, 0, 0, 182, 192, 843, 0, 0, 3, 0, 429, 0, 0, 6, 77, 0, 0, 0, 1278, 0, 4, 11, 13, 2339, 0, 0, 9, 4060, 0, 802, 4040, 0, 0, 32, 12, 2, 4, 14, 0, 4085, 6, 8, 16] payload_rows_per_dst=[5018, 7630, 7892, 8174, 8172, 4704, 5096, 8192] dup_rows_per_dst=[122, 2, 3176, 3138, 2670, 48, 1424, 78] combine_rows_per_src=[5018, 7630, 7892, 8174, 8172, 4704, 5096, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59294 payload_rows_remote=51102 dup_rows=6242 dup_rows_remote=4532 dedup_ratio=0.095245 router_tokens_per_expert=[0, 0, 0, 4, 0, 2026, 2, 0, 4079, 0, 0, 4087, 0, 0, 0, 0, 0, 7, 0, 3964, 0, 0, 0, 0, 43, 47, 709, 0, 0, 0, 0, 332, 0, 0, 1, 0, 0, 0, 0, 854, 0, 0, 0, 0, 3571, 0, 0, 0, 4096, 0, 1376, 3849, 0, 0, 2, 0, 0, 0, 0, 2, 3717, 0, 0, 0] payload_rows_per_dst=[4056, 8158, 7938, 8188, 8192, 7142, 8182, 7438] dup_rows_per_dst=[8, 0, 4, 2248, 1710, 0, 2272, 0] combine_rows_per_src=[4056, 8158, 7938, 8188, 8192, 7142, 8182, 7438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58068 payload_rows_remote=51512 dup_rows=7468 dup_rows_remote=7354 dedup_ratio=0.113953 router_tokens_per_expert=[0, 0, 2, 0, 0, 3406, 1, 0, 4003, 5, 0, 4041, 0, 0, 0, 0, 0, 36, 0, 2200, 0, 0, 0, 0, 594, 18, 2, 0, 0, 0, 0, 269, 0, 0, 573, 0, 0, 0, 0, 644, 0, 0, 0, 0, 3277, 0, 0, 0, 4096, 0, 1656, 3964, 0, 0, 0, 0, 0, 4, 0, 0, 3977, 0, 0, 0] payload_rows_per_dst=[6812, 7898, 4436, 8122, 8192, 6556, 8098, 7954] dup_rows_per_dst=[6, 2, 36, 1726, 2434, 114, 3142, 8] combine_rows_per_src=[6812, 7898, 4436, 8122, 8192, 6556, 8098, 7954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=47968 payload_rows_remote=40380 dup_rows=17568 dup_rows_remote=14286 dedup_ratio=0.268066 router_tokens_per_expert=[0, 1, 0, 0, 0, 3467, 5, 0, 3995, 9, 0, 4026, 0, 0, 1, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 698, 404, 28, 0, 0, 0, 0, 823, 0, 0, 3, 0, 0, 0, 0, 497, 0, 0, 0, 0, 2989, 0, 0, 0, 4096, 0, 91, 3752, 0, 0, 0, 0, 0, 492, 0, 0, 3318, 0, 0, 0] payload_rows_per_dst=[6944, 18, 8146, 8192, 1000, 7888, 8192, 7588] dup_rows_per_dst=[2, 2, 0, 3766, 0, 4542, 5974, 3282] combine_rows_per_src=[6944, 18, 8146, 8192, 1000, 7888, 8192, 7588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=175448064 dispatch_meta_bytes=206520 combine_payload_bytes=175448064 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=175654584 ingress_bytes=250945536 link_bytes=250945536 token_bytes=351102648 weight_movement_bytes=75497472 total_nvlink_bytes=426600120 weight_to_token_ratio=0.215030 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.909405577e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=48704 payload_rows_remote=41784 dup_rows=16832 dup_rows_remote=12618 dedup_ratio=0.256836 router_tokens_per_expert=[0, 11, 0, 2, 0, 3642, 1, 0, 4036, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3166, 0, 0, 0, 0, 7, 488, 48, 0, 0, 0, 0, 1618, 0, 0, 6, 0, 0, 0, 0, 1389, 0, 0, 0, 0, 2957, 0, 0, 0, 4096, 0, 37, 3441, 0, 0, 1, 0, 0, 93, 0, 0, 3626, 0, 0, 0] payload_rows_per_dst=[7296, 14, 6332, 8192, 4354, 8158, 6920, 7438] dup_rows_per_dst=[16, 0, 0, 4322, 2452, 5828, 4214, 0] combine_rows_per_src=[7296, 14, 6332, 8192, 4354, 8158, 6920, 7438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=191299584 dispatch_meta_bytes=216896 combine_payload_bytes=191299584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=191516480 ingress_bytes=216465408 link_bytes=216465408 token_bytes=382816064 weight_movement_bytes=25165824 total_nvlink_bytes=407981888 weight_to_token_ratio=0.065739 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.647051643e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=209313792 dispatch_meta_bytes=222536 combine_payload_bytes=209313792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=209536328 ingress_bytes=234479616 link_bytes=234479616 token_bytes=418850120 weight_movement_bytes=25165824 total_nvlink_bytes=444015944 weight_to_token_ratio=0.060083 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=26624 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784118952e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=210993152 dispatch_meta_bytes=235464 combine_payload_bytes=210993152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=211228616 ingress_bytes=236158976 link_bytes=236158976 token_bytes=422221768 weight_movement_bytes=25165824 total_nvlink_bytes=447387592 weight_to_token_ratio=0.059603 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=25472 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.796896941e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=165396480 dispatch_meta_bytes=218664 combine_payload_bytes=165396480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=165615144 ingress_bytes=190562304 link_bytes=190562304 token_bytes=331011624 weight_movement_bytes=25165824 total_nvlink_bytes=356177448 weight_to_token_ratio=0.076027 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.449958951e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=171147264 dispatch_meta_bytes=217608 combine_payload_bytes=171147264 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=171364872 ingress_bytes=296976384 link_bytes=296976384 token_bytes=342512136 weight_movement_bytes=125829120 total_nvlink_bytes=468341256 weight_to_token_ratio=0.367371 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=15872 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.259647145e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52182 payload_rows_remote=44026 dup_rows=13354 dup_rows_remote=5594 dedup_ratio=0.203766 router_tokens_per_expert=[0, 121, 9, 0, 0, 2775, 0, 0, 3438, 72, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 1, 0, 0, 7, 1472, 18, 0, 0, 0, 0, 19, 0, 0, 90, 0, 0, 0, 0, 569, 0, 0, 0, 0, 3864, 0, 0, 0, 4096, 0, 122, 4014, 0, 0, 0, 0, 0, 17, 0, 0, 3916, 0, 0, 0] payload_rows_per_dst=[5814, 8174, 8156, 6042, 8192, 7728, 244, 7832] dup_rows_per_dst=[222, 3664, 7760, 356, 1318, 0, 0, 34] combine_rows_per_src=[5814, 8174, 8156, 6042, 8192, 7728, 244, 7832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52962 payload_rows_remote=44858 dup_rows=12574 dup_rows_remote=8316 dedup_ratio=0.191864 router_tokens_per_expert=[0, 1, 0, 0, 0, 2128, 0, 0, 3938, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3184, 0, 0, 0, 0, 42, 158, 48, 0, 0, 0, 0, 811, 0, 0, 0, 2, 0, 0, 0, 186, 0, 0, 0, 0, 2408, 0, 0, 0, 4095, 0, 3579, 4052, 0, 0, 2, 0, 0, 30, 0, 0, 4008, 0, 0, 0] payload_rows_per_dst=[8104, 8192, 6368, 2072, 8190, 4816, 7160, 8060] dup_rows_per_dst=[4258, 7876, 0, 46, 376, 0, 2, 16] combine_rows_per_src=[8104, 8192, 6368, 2072, 8190, 4816, 7160, 8060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57532 payload_rows_remote=49990 dup_rows=8004 dup_rows_remote=8004 dedup_ratio=0.122131 router_tokens_per_expert=[0, 0, 0, 0, 4096, 1, 4062, 52, 0, 0, 4096, 0, 3771, 0, 0, 0, 0, 3006, 0, 0, 3212, 4010, 0, 0, 27, 0, 0, 0, 15, 0, 1679, 0, 3987, 623, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 52, 0] payload_rows_per_dst=[8160, 7542, 8048, 3406, 7974, 8192, 8192, 6018] dup_rows_per_dst=[70, 0, 6396, 36, 1246, 14, 144, 98] combine_rows_per_src=[8160, 7542, 8048, 3406, 7974, 8192, 8192, 6018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=180330496 dispatch_meta_bytes=198480 combine_payload_bytes=180330496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=180528976 ingress_bytes=205496320 link_bytes=205496320 token_bytes=360859472 weight_movement_bytes=25165824 total_nvlink_bytes=386025296 weight_to_token_ratio=0.069739 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27264 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.563589557e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=183738368 dispatch_meta_bytes=212696 combine_payload_bytes=183738368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=183951064 ingress_bytes=208904192 link_bytes=208904192 token_bytes=367689432 weight_movement_bytes=25165824 total_nvlink_bytes=392855256 weight_to_token_ratio=0.068443 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589519526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56806 payload_rows_remote=50478 dup_rows=8730 dup_rows_remote=8676 dedup_ratio=0.133209 router_tokens_per_expert=[0, 0, 0, 0, 4047, 0, 4082, 106, 0, 0, 4077, 9, 2157, 9, 0, 0, 0, 4060, 10, 0, 3489, 3948, 1, 0, 29, 0, 0, 0, 0, 10, 3127, 25, 2662, 0, 1, 0, 0, 0, 6, 1, 0, 0, 0, 39, 0, 14, 10, 0, 0, 7, 12, 0, 11, 0, 11, 749, 0, 0, 0, 31, 0, 0, 28, 0] payload_rows_per_dst=[8164, 4332, 8162, 6328, 5340, 8130, 8192, 8158] dup_rows_per_dst=[212, 18, 6734, 54, 0, 90, 1542, 80] combine_rows_per_src=[8164, 4332, 8162, 6328, 5340, 8130, 8192, 8158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=61198 payload_rows_remote=54784 dup_rows=4338 dup_rows_remote=4176 dedup_ratio=0.066193 router_tokens_per_expert=[0, 0, 0, 0, 4088, 0, 4088, 170, 0, 0, 4096, 0, 3980, 0, 0, 0, 2, 3766, 0, 0, 3940, 4042, 0, 0, 404, 0, 0, 0, 0, 0, 807, 0, 3202, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 39, 0] payload_rows_per_dst=[8182, 7960, 8034, 6706, 6414, 8176, 8192, 7534] dup_rows_per_dst=[334, 0, 1976, 1674, 162, 0, 116, 76] combine_rows_per_src=[8182, 7960, 8034, 6706, 6414, 8176, 8192, 7534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58446 payload_rows_remote=50264 dup_rows=7090 dup_rows_remote=6824 dedup_ratio=0.108185 router_tokens_per_expert=[0, 0, 0, 0, 4081, 0, 4092, 8, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 4090, 0, 0, 3736, 3795, 0, 0, 192, 0, 0, 0, 0, 0, 1440, 0, 3019, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8142, 1188, 8186, 8184, 8182, 8192, 8180] dup_rows_per_dst=[8, 0, 0, 2668, 4140, 266, 8, 0] combine_rows_per_src=[8192, 8142, 1188, 8186, 8184, 8182, 8192, 8180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=52122 payload_rows_remote=43954 dup_rows=13414 dup_rows_remote=10658 dedup_ratio=0.204681 router_tokens_per_expert=[0, 0, 0, 0, 4071, 0, 4094, 0, 0, 0, 4096, 0, 3675, 0, 0, 1, 10, 4084, 0, 0, 3551, 3922, 0, 0, 38, 0, 0, 0, 20, 0, 2479, 0, 2398, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4256, 7758, 20, 8190, 7396, 8142, 8192, 8168] dup_rows_per_dst=[0, 3526, 0, 4728, 2332, 66, 6, 2756] combine_rows_per_src=[4256, 7758, 20, 8190, 7396, 8142, 8192, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=204759040 dispatch_meta_bytes=231976 combine_payload_bytes=204759040 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=204991016 ingress_bytes=255090688 link_bytes=255090688 token_bytes=409750056 weight_movement_bytes=50331648 total_nvlink_bytes=460081704 weight_to_token_ratio=0.122835 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=29696 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.940945395e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57198 payload_rows_remote=49006 dup_rows=8338 dup_rows_remote=8338 dedup_ratio=0.127228 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 4043, 0, 0, 3919, 4049, 0, 0, 26, 0, 0, 0, 0, 0, 1885, 0, 2440, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8152, 0, 8192, 8192, 8192, 8192, 8086] dup_rows_per_dst=[12, 0, 0, 3728, 4596, 2, 0, 0] combine_rows_per_src=[8192, 8152, 0, 8192, 8192, 8192, 8192, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=206757888 dispatch_meta_bytes=236616 combine_payload_bytes=206757888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206994504 ingress_bytes=231923712 link_bytes=231923712 token_bytes=413752392 weight_movement_bytes=25165824 total_nvlink_bytes=438918216 weight_to_token_ratio=0.060823 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17152 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.764671475e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=224395264 dispatch_meta_bytes=235840 combine_payload_bytes=224395264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=224631104 ingress_bytes=249561088 link_bytes=249561088 token_bytes=449026368 weight_movement_bytes=25165824 total_nvlink_bytes=474192192 weight_to_token_ratio=0.056045 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=22528 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.898871528e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=205881344 dispatch_meta_bytes=228352 combine_payload_bytes=205881344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=206109696 ingress_bytes=231047168 link_bytes=231047168 token_bytes=411991040 weight_movement_bytes=25165824 total_nvlink_bytes=437156864 weight_to_token_ratio=0.061083 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.758001988e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=180035584 dispatch_meta_bytes=218448 combine_payload_bytes=180035584 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=180254032 ingress_bytes=230367232 link_bytes=230367232 token_bytes=360289616 weight_movement_bytes=50331648 total_nvlink_bytes=410621264 weight_to_token_ratio=0.139698 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31360 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.752828461e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59018 payload_rows_remote=51658 dup_rows=6518 dup_rows_remote=1572 dedup_ratio=0.099457 router_tokens_per_expert=[0, 0, 0, 0, 4093, 0, 4089, 457, 0, 0, 4096, 0, 3971, 0, 0, 0, 3, 4095, 0, 0, 3427, 2723, 0, 0, 93, 0, 0, 0, 0, 0, 1895, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7942, 7360, 3868, 7090, 8186, 8192, 8190] dup_rows_per_dst=[902, 0, 4946, 108, 0, 80, 482, 0] combine_rows_per_src=[8190, 7942, 7360, 3868, 7090, 8186, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=200728576 dispatch_meta_bytes=229376 combine_payload_bytes=200728576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200957952 ingress_bytes=225894400 link_bytes=225894400 token_bytes=401686528 weight_movement_bytes=25165824 total_nvlink_bytes=426852352 weight_to_token_ratio=0.062650 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.718795377e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56332 payload_rows_remote=48142 dup_rows=9204 dup_rows_remote=6732 dedup_ratio=0.140442 router_tokens_per_expert=[0, 0, 0, 0, 4094, 0, 3911, 337, 0, 0, 4096, 0, 3906, 0, 0, 0, 0, 4094, 0, 0, 3219, 4014, 0, 0, 27, 0, 0, 0, 0, 0, 3805, 0, 1158, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8190, 8126, 8086, 7630, 2400, 7818, 5894, 8188] dup_rows_per_dst=[2472, 2038, 4586, 34, 70, 0, 0, 4] combine_rows_per_src=[8190, 8126, 8086, 7630, 2400, 7818, 5894, 8188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57428 payload_rows_remote=49350 dup_rows=8108 dup_rows_remote=8108 dedup_ratio=0.123718 router_tokens_per_expert=[69, 2177, 0, 2288, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 2714, 1, 4077, 0, 0, 76, 0, 0, 0, 1136, 0, 0, 0, 0, 0, 3906, 0, 4, 0, 0, 46, 0, 11, 0, 0, 4, 0, 0, 0, 4024, 61, 0, 3987, 0, 108, 0] payload_rows_per_dst=[7108, 8078, 7938, 7872, 2414, 7812, 8192, 8014] dup_rows_per_dst=[2590, 0, 0, 5082, 10, 8, 120, 298] combine_rows_per_src=[7108, 8078, 7938, 7872, 2414, 7812, 8192, 8014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=211591168 dispatch_meta_bytes=212920 combine_payload_bytes=211591168 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=211804088 ingress_bytes=337420288 link_bytes=337420288 token_bytes=423395256 weight_movement_bytes=125829120 total_nvlink_bytes=549224376 weight_to_token_ratio=0.297191 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.567378524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62288 payload_rows_remote=54252 dup_rows=3248 dup_rows_remote=3002 dedup_ratio=0.049561 router_tokens_per_expert=[0, 636, 9, 4050, 4, 12, 0, 0, 11, 219, 5, 0, 0, 0, 0, 13, 0, 4070, 0, 0, 4086, 0, 162, 0, 7, 29, 0, 10, 150, 0, 3945, 0, 0, 76, 0, 0, 0, 3186, 0, 0, 3, 0, 0, 3045, 0, 916, 13, 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 4076, 4, 0, 3933, 0, 79, 4] payload_rows_per_dst=[8122, 8178, 8176, 8036, 6464, 7222, 8166, 7924] dup_rows_per_dst=[1300, 470, 320, 246, 60, 732, 4, 116] combine_rows_per_src=[8122, 8178, 8176, 8036, 6464, 7222, 8166, 7924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=197189632 dispatch_meta_bytes=219496 combine_payload_bytes=197189632 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=197409128 ingress_bytes=272687104 link_bytes=272687104 token_bytes=394598760 weight_movement_bytes=75497472 total_nvlink_bytes=470096232 weight_to_token_ratio=0.191327 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=30592 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.074833789e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57454 payload_rows_remote=49632 dup_rows=8082 dup_rows_remote=6924 dedup_ratio=0.123322 router_tokens_per_expert=[11, 158, 0, 3127, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3674, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 3875, 0, 0, 2, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 1585, 0, 24, 0, 0, 40, 0, 4, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[6344, 8182, 7348, 8184, 7822, 3190, 8192, 8192] dup_rows_per_dst=[248, 2, 0, 6558, 1158, 28, 88, 0] combine_rows_per_src=[6344, 8182, 7348, 8184, 7822, 3190, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54950 payload_rows_remote=49312 dup_rows=10586 dup_rows_remote=10576 dedup_ratio=0.161530 router_tokens_per_expert=[4, 399, 0, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4075, 0, 21, 0, 0, 9, 0, 0, 2337, 0, 4088, 0, 0, 1076, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 2818, 0, 6, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0] payload_rows_per_dst=[3718, 8180, 8160, 4690, 8180, 5638, 8192, 8192] dup_rows_per_dst=[166, 0, 36, 2, 9880, 10, 492, 0] combine_rows_per_src=[3718, 8180, 8160, 4690, 8180, 5638, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=54770 payload_rows_remote=46588 dup_rows=10766 dup_rows_remote=10272 dedup_ratio=0.164276 router_tokens_per_expert=[91, 272, 0, 1988, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 3640, 0, 2, 0, 0, 0, 0, 0, 1385, 11, 4044, 0, 0, 1777, 0, 1, 0, 3147, 0, 0, 0, 0, 0, 3990, 1, 71, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 4091, 0, 1, 0] payload_rows_per_dst=[4392, 7702, 7332, 2792, 7992, 8186, 8192, 8182] dup_rows_per_dst=[310, 66, 0, 0, 1858, 8026, 12, 494] combine_rows_per_src=[4392, 7702, 7332, 2792, 7992, 8186, 8192, 8182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=202137600 dispatch_meta_bytes=229832 combine_payload_bytes=202137600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202367432 ingress_bytes=227303424 link_bytes=227303424 token_bytes=404505032 weight_movement_bytes=25165824 total_nvlink_bytes=429670856 weight_to_token_ratio=0.062214 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.729516422e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=222216192 dispatch_meta_bytes=229016 combine_payload_bytes=222216192 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=222445208 ingress_bytes=322879488 link_bytes=322879488 token_bytes=444661400 weight_movement_bytes=100663296 total_nvlink_bytes=545324696 weight_to_token_ratio=0.226382 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17536 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.456739837e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53192 payload_rows_remote=45000 dup_rows=12344 dup_rows_remote=12252 dedup_ratio=0.188354 router_tokens_per_expert=[4, 380, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4057, 0, 0, 0, 0, 90, 0, 0, 753, 0, 3875, 0, 0, 262, 0, 0, 0, 3575, 0, 0, 0, 0, 0, 4094, 0, 1882, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4095, 0, 545, 0] payload_rows_per_dst=[2570, 8192, 8116, 1674, 8066, 8192, 8192, 8190] dup_rows_per_dst=[16, 0, 8, 12, 5220, 5898, 92, 1098] combine_rows_per_src=[2570, 8192, 8116, 1674, 8066, 8192, 8192, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=203292672 dispatch_meta_bytes=226224 combine_payload_bytes=203292672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=203518896 ingress_bytes=228458496 link_bytes=228458496 token_bytes=406811568 weight_movement_bytes=25165824 total_nvlink_bytes=431977392 weight_to_token_ratio=0.061861 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=20864 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.738305185e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=201981952 dispatch_meta_bytes=239552 combine_payload_bytes=201981952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=202221504 ingress_bytes=227147776 link_bytes=227147776 token_bytes=404203456 weight_movement_bytes=25165824 total_nvlink_bytes=429369280 weight_to_token_ratio=0.062260 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.728332121e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=190824448 dispatch_meta_bytes=227440 combine_payload_bytes=190824448 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=191051888 ingress_bytes=266321920 link_bytes=266321920 token_bytes=381876336 weight_movement_bytes=75497472 total_nvlink_bytes=457373808 weight_to_token_ratio=0.197701 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.026402093e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53600 payload_rows_remote=45550 dup_rows=11936 dup_rows_remote=11928 dedup_ratio=0.182129 router_tokens_per_expert=[46, 2676, 0, 4068, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 79, 3950, 0, 0, 0, 0, 0, 0, 0, 3269, 0, 4021, 0, 0, 522, 0, 0, 0, 377, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 4093, 0, 0, 0] payload_rows_per_dst=[8184, 8180, 8050, 8112, 1762, 2934, 8192, 8186] dup_rows_per_dst=[5396, 12, 8, 6468, 36, 0, 10, 6] combine_rows_per_src=[8184, 8180, 8050, 8112, 1762, 2934, 8192, 8186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56374 payload_rows_remote=48198 dup_rows=9162 dup_rows_remote=2266 dedup_ratio=0.139801 router_tokens_per_expert=[0, 434, 0, 3268, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 2746, 0, 3834, 0, 0, 697, 0, 0, 0, 1610, 0, 0, 0, 0, 0, 3823, 0, 7, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8192, 5680, 4444, 7646, 7400, 6644] dup_rows_per_dst=[6896, 102, 1852, 128, 170, 14, 0, 0] combine_rows_per_src=[8176, 8192, 8192, 5680, 4444, 7646, 7400, 6644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=184320000 dispatch_meta_bytes=229008 combine_payload_bytes=184320000 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=184549008 ingress_bytes=209485824 link_bytes=209485824 token_bytes=368869008 weight_movement_bytes=25165824 total_nvlink_bytes=394034832 weight_to_token_ratio=0.068224 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593945073e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=51160 payload_rows_remote=42996 dup_rows=14376 dup_rows_remote=14376 dedup_ratio=0.219360 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2450, 0, 4096, 0, 0, 2187, 785, 4, 3, 0, 0, 0, 149, 0, 0, 0, 2767, 0, 0, 235, 3178, 0, 1119, 0, 0, 3723, 0, 3738, 4082, 126, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8038, 8164, 8192, 8192, 8164, 2486, 7710, 214] dup_rows_per_dst=[0, 0, 4900, 5212, 4024, 222, 18, 0] combine_rows_per_src=[8038, 8164, 8192, 8192, 8164, 2486, 7710, 214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=186572800 dispatch_meta_bytes=229912 combine_payload_bytes=186572800 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=186802712 ingress_bytes=236904448 link_bytes=236904448 token_bytes=373375512 weight_movement_bytes=50331648 total_nvlink_bytes=423707160 weight_to_token_ratio=0.134802 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.802569121e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=59890 payload_rows_remote=52410 dup_rows=5646 dup_rows_remote=5158 dedup_ratio=0.086151 router_tokens_per_expert=[12, 3, 0, 4, 0, 3, 4084, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4, 0, 3822, 0, 4073, 35, 5, 283, 73, 0, 0, 10, 10, 16, 686, 0, 23, 7, 4060, 11, 10, 3, 2401, 5, 104, 0, 0, 3568, 0, 3454, 4083, 0, 0, 0, 0, 0, 100, 1799, 0, 3, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7652, 7480, 8160, 4988, 7076, 8150] dup_rows_per_dst=[20, 2, 0, 488, 1466, 38, 32, 3600] combine_rows_per_src=[8192, 8192, 7652, 7480, 8160, 4988, 7076, 8150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=197419008 dispatch_meta_bytes=201856 combine_payload_bytes=197419008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=197620864 ingress_bytes=222584832 link_bytes=222584832 token_bytes=395039872 weight_movement_bytes=25165824 total_nvlink_bytes=420205696 weight_to_token_ratio=0.063705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.693613389e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58466 payload_rows_remote=56260 dup_rows=7070 dup_rows_remote=7070 dedup_ratio=0.107880 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 0, 4096, 0, 0, 2496, 98, 5, 1, 0, 0, 0, 56, 0, 0, 0, 1047, 0, 0, 11, 3854, 0, 366, 0, 0, 4081, 0, 4088, 4096, 21, 0, 0, 0, 0, 1, 574, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8136, 8192, 7618, 8190, 2206, 7754, 8178, 8192] dup_rows_per_dst=[0, 0, 0, 5172, 0, 708, 42, 1148] combine_rows_per_src=[8136, 8192, 7618, 8190, 2206, 7754, 8178, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58996 payload_rows_remote=50910 dup_rows=6540 dup_rows_remote=2860 dedup_ratio=0.099792 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3165, 0, 4096, 0, 0, 1437, 7, 0, 0, 0, 0, 0, 141, 0, 0, 0, 3349, 0, 0, 556, 2545, 0, 75, 0, 0, 4080, 0, 3995, 4096, 123, 0, 0, 0, 0, 3, 1004, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6330, 5162, 6766, 8086, 8076, 8192] dup_rows_per_dst=[0, 0, 0, 472, 214, 3680, 166, 2008] combine_rows_per_src=[8192, 8192, 6330, 5162, 6766, 8086, 8076, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=49794 payload_rows_remote=41602 dup_rows=15742 dup_rows_remote=15698 dedup_ratio=0.240204 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4061, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3001, 0, 4096, 0, 0, 15, 655, 12, 11, 0, 0, 0, 31, 0, 0, 0, 3814, 0, 0, 531, 4052, 0, 278, 0, 0, 4065, 0, 4039, 4077, 5, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 3] payload_rows_per_dst=[8126, 8156, 0, 1364, 7644, 8192, 8120, 8192] dup_rows_per_dst=[0, 0, 0, 22, 46, 10470, 5160, 44] combine_rows_per_src=[8126, 8156, 0, 1364, 7644, 8192, 8120, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=176111616 dispatch_meta_bytes=229488 combine_payload_bytes=176111616 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=176341104 ingress_bytes=226443264 link_bytes=226443264 token_bytes=352452720 weight_movement_bytes=50331648 total_nvlink_bytes=402784368 weight_to_token_ratio=0.142804 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.722971598e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=214671360 dispatch_meta_bytes=230272 combine_payload_bytes=214671360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=214901632 ingress_bytes=239837184 link_bytes=239837184 token_bytes=429572992 weight_movement_bytes=25165824 total_nvlink_bytes=454738816 weight_to_token_ratio=0.058583 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.824883854e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58172 payload_rows_remote=50576 dup_rows=7364 dup_rows_remote=7132 dedup_ratio=0.112366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4058, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 4096, 0, 0, 3519, 10, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1928, 0, 0, 0, 2597, 0, 4, 0, 0, 4092, 0, 3616, 4096, 16, 0, 0, 0, 0, 282, 551, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8120, 8192, 6982, 7040, 3858, 8192, 7596, 8192] dup_rows_per_dst=[0, 0, 0, 18, 4, 6008, 232, 1102] combine_rows_per_src=[8120, 8192, 6982, 7040, 3858, 8192, 7596, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=230440960 dispatch_meta_bytes=253320 combine_payload_bytes=230440960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230694280 ingress_bytes=255606784 link_bytes=255606784 token_bytes=461135240 weight_movement_bytes=25165824 total_nvlink_bytes=486301064 weight_to_token_ratio=0.054574 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24192 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.944872289e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=170401792 dispatch_meta_bytes=229200 combine_payload_bytes=170401792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=170630992 ingress_bytes=195567616 link_bytes=195567616 token_bytes=341032784 weight_movement_bytes=25165824 total_nvlink_bytes=366198608 weight_to_token_ratio=0.073793 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=24448 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.488043592e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=208527360 dispatch_meta_bytes=215080 combine_payload_bytes=208527360 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=208742440 ingress_bytes=284024832 link_bytes=284024832 token_bytes=417269800 weight_movement_bytes=75497472 total_nvlink_bytes=492767272 weight_to_token_ratio=0.180932 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17280 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.161100799e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=58234 payload_rows_remote=50502 dup_rows=7302 dup_rows_remote=6962 dedup_ratio=0.111420 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 4091, 0, 0, 225, 0, 0, 1, 0, 0, 0, 13, 0, 0, 0, 3131, 0, 0, 201, 3992, 0, 163, 0, 0, 4092, 0, 3156, 4095, 5, 0, 0, 0, 0, 0, 1637, 0, 0, 0, 0, 15, 0, 1] payload_rows_per_dst=[8184, 8190, 7732, 8184, 7568, 4120, 6322, 7934] dup_rows_per_dst=[0, 0, 340, 452, 3134, 178, 0, 3198] combine_rows_per_src=[8184, 8190, 7732, 8184, 7568, 4120, 6322, 7934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=53244 payload_rows_remote=45128 dup_rows=12292 dup_rows_remote=11880 dedup_ratio=0.187561 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4052, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 4096, 0, 0, 27, 59, 0, 0, 0, 0, 0, 39, 0, 0, 0, 4040, 0, 0, 4, 2062, 0, 203, 0, 0, 3958, 0, 3390, 4091, 22, 0, 0, 0, 0, 29, 2683, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[8116, 8192, 8192, 8008, 8190, 412, 6764, 5370] dup_rows_per_dst=[412, 64, 7598, 80, 4092, 2, 44, 0] combine_rows_per_src=[8116, 8192, 8192, 8008, 8190, 412, 6764, 5370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=207159296 dispatch_meta_bytes=230832 combine_payload_bytes=207159296 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=207390128 ingress_bytes=282656768 link_bytes=282656768 token_bytes=414549424 weight_movement_bytes=75497472 total_nvlink_bytes=490046896 weight_to_token_ratio=0.182119 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.150691413e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64480 payload_rows_remote=56288 dup_rows=1056 dup_rows_remote=884 dedup_ratio=0.016113 router_tokens_per_expert=[0, 0, 0, 0, 3603, 4096, 0, 0, 86, 7, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 116, 0, 0, 0, 2, 0, 0, 420, 0, 4089, 3974, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7206, 8192, 8184, 8192, 8192, 8144, 8192, 8178] dup_rows_per_dst=[0, 172, 0, 2, 234, 648, 0, 0] combine_rows_per_src=[7206, 8192, 8184, 8192, 8192, 8144, 8192, 8178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65324 payload_rows_remote=57132 dup_rows=212 dup_rows_remote=164 dedup_ratio=0.003235 router_tokens_per_expert=[0, 0, 0, 0, 4079, 3830, 3, 10, 0, 0, 5, 1, 0, 0, 0, 0, 0, 4082, 0, 0, 4081, 7, 0, 0, 0, 0, 0, 0, 0, 11, 3, 276, 4083, 0, 0, 8, 0, 48, 0, 6, 0, 0, 5, 1, 0, 4084, 4081, 13, 0, 0, 0, 0, 0, 0, 4016, 0, 13, 0, 0, 10, 0, 0, 6, 6] payload_rows_per_dst=[8178, 8174, 8176, 8192, 8192, 8188, 8032, 8192] dup_rows_per_dst=[6, 2, 0, 48, 98, 12, 0, 46] combine_rows_per_src=[8178, 8174, 8176, 8192, 8192, 8188, 8032, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=206856192 dispatch_meta_bytes=229856 combine_payload_bytes=206856192 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=207086048 ingress_bytes=307519488 link_bytes=307519488 token_bytes=413942240 weight_movement_bytes=100663296 total_nvlink_bytes=514605536 weight_to_token_ratio=0.243182 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24448 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.339867985e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=184844288 dispatch_meta_bytes=228032 combine_payload_bytes=184844288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=185072320 ingress_bytes=210010112 link_bytes=210010112 token_bytes=369916608 weight_movement_bytes=25165824 total_nvlink_bytes=395082432 weight_to_token_ratio=0.068031 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597934299e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64930 payload_rows_remote=56738 dup_rows=606 dup_rows_remote=606 dedup_ratio=0.009247 router_tokens_per_expert=[0, 0, 0, 0, 3840, 4096, 0, 0, 12, 4, 0, 9, 0, 0, 0, 0, 0, 4089, 0, 0, 4049, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 294, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7680, 8192, 8098, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 36, 0, 2, 0, 568, 0, 0] combine_rows_per_src=[7680, 8192, 8098, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=63764 payload_rows_remote=55580 dup_rows=1772 dup_rows_remote=1748 dedup_ratio=0.027039 router_tokens_per_expert=[0, 0, 0, 0, 4073, 4096, 0, 0, 46, 0, 0, 19, 0, 0, 0, 0, 2, 4067, 0, 0, 4092, 0, 0, 0, 4, 0, 12, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 197, 0, 0, 0, 4, 0, 0, 20, 0, 4092, 4037, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8146, 7804, 8188, 8192, 7310, 8108, 7832, 8184] dup_rows_per_dst=[0, 78, 162, 32, 386, 1022, 68, 24] combine_rows_per_src=[8146, 7804, 8188, 8192, 7310, 8108, 7832, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4088, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=230555648 dispatch_meta_bytes=228688 combine_payload_bytes=230555648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=230784336 ingress_bytes=255721472 link_bytes=255721472 token_bytes=461339984 weight_movement_bytes=25165824 total_nvlink_bytes=486505808 weight_to_token_ratio=0.054549 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.945744932e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234012672 dispatch_meta_bytes=229184 combine_payload_bytes=234012672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=234241856 ingress_bytes=259178496 link_bytes=259178496 token_bytes=468254528 weight_movement_bytes=25165824 total_nvlink_bytes=493420352 weight_to_token_ratio=0.053744 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.972048890e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=64592 payload_rows_remote=56416 dup_rows=944 dup_rows_remote=944 dedup_ratio=0.014404 router_tokens_per_expert=[0, 0, 0, 0, 3682, 4075, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 24, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 633, 0, 4076, 3884, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7364, 8186, 8184, 8150, 8192, 8188, 8176, 8152] dup_rows_per_dst=[0, 2, 28, 4, 64, 846, 0, 0] combine_rows_per_src=[7364, 8186, 8184, 8150, 8192, 8188, 8176, 8152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=232398848 dispatch_meta_bytes=229376 combine_payload_bytes=232398848 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=232628224 ingress_bytes=282730496 link_bytes=282730496 token_bytes=465027072 weight_movement_bytes=50331648 total_nvlink_bytes=515358720 weight_to_token_ratio=0.108234 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32256 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.151252398e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=227655680 dispatch_meta_bytes=229312 combine_payload_bytes=227655680 prefetch_bytes=50331648 reduce_bytes=0 egress_bytes=227884992 ingress_bytes=277987328 link_bytes=277987328 token_bytes=455540672 weight_movement_bytes=50331648 total_nvlink_bytes=505872320 weight_to_token_ratio=0.110488 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.115162369e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234864640 dispatch_meta_bytes=229376 combine_payload_bytes=234864640 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=235094016 ingress_bytes=335527936 link_bytes=335527936 token_bytes=469958656 weight_movement_bytes=100663296 total_nvlink_bytes=570621952 weight_to_token_ratio=0.214196 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.552979912e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=65304 payload_rows_remote=57112 dup_rows=232 dup_rows_remote=232 dedup_ratio=0.003540 router_tokens_per_expert=[0, 0, 0, 8, 4003, 4096, 0, 0, 9, 11, 0, 25, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 453, 0, 4092, 3671, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8022, 8190, 8192, 8192, 8192, 8140, 8192, 8184] dup_rows_per_dst=[0, 70, 0, 0, 54, 108, 0, 0] combine_rows_per_src=[8022, 8190, 8192, 8192, 8192, 8140, 8192, 8184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=62378 payload_rows_remote=54218 dup_rows=3158 dup_rows_remote=576 dedup_ratio=0.048187 router_tokens_per_expert=[0, 0, 0, 0, 3826, 4096, 0, 0, 26, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 45, 0, 4096, 4069, 0, 0, 232, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8160, 8192, 8132, 8150, 8192, 5168, 8192, 8192] dup_rows_per_dst=[2582, 54, 0, 0, 46, 12, 464, 0] combine_rows_per_src=[8160, 8192, 8132, 8150, 8192, 5168, 8192, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=231079936 dispatch_meta_bytes=229440 combine_payload_bytes=231079936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=231309376 ingress_bytes=256245760 link_bytes=256245760 token_bytes=462389312 weight_movement_bytes=25165824 total_nvlink_bytes=487555136 weight_to_token_ratio=0.054426 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.949734158e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=60332 payload_rows_remote=52152 dup_rows=5204 dup_rows_remote=5172 dedup_ratio=0.079407 router_tokens_per_expert=[0, 40, 4086, 0, 19, 0, 0, 0, 4078, 0, 13, 3, 0, 2, 4047, 0, 0, 0, 0, 0, 4084, 5, 0, 0, 0, 10, 11, 0, 0, 7, 0, 0, 0, 0, 0, 4078, 20, 9, 15, 0, 11, 0, 2994, 0, 12, 26, 1, 7, 1, 0, 1082, 61, 0, 2, 0, 0, 0, 29, 0, 0, 0, 3935, 4080, 0] payload_rows_per_dst=[8172, 8126, 8178, 8180, 8192, 8110, 8166, 3208] dup_rows_per_dst=[118, 4, 0, 32, 52, 2712, 2286, 0] combine_rows_per_src=[8172, 8126, 8178, 8180, 8192, 8110, 8166, 3208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=55794 payload_rows_remote=47602 dup_rows=9742 dup_rows_remote=9742 dedup_ratio=0.148651 router_tokens_per_expert=[0, 0, 3422, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 525, 9, 0, 28, 0, 0, 0, 0, 3642, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[6844, 8192, 8086, 8192, 8096, 8192, 8192, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 1124, 8618, 0] combine_rows_per_src=[6844, 8192, 8086, 8192, 8096, 8192, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=233930752 dispatch_meta_bytes=229376 combine_payload_bytes=233930752 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=234160128 ingress_bytes=309428224 link_bytes=309428224 token_bytes=468090880 weight_movement_bytes=75497472 total_nvlink_bytes=543588352 weight_to_token_ratio=0.161288 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.354391260e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=222076928 dispatch_meta_bytes=219176 combine_payload_bytes=222076928 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=222296104 ingress_bytes=272408576 link_bytes=272408576 token_bytes=444373032 weight_movement_bytes=50331648 total_nvlink_bytes=494704680 weight_to_token_ratio=0.113264 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.072714513e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57948 payload_rows_remote=49782 dup_rows=7588 dup_rows_remote=7588 dedup_ratio=0.115784 router_tokens_per_expert=[0, 209, 3796, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 2597, 4, 0, 45, 0, 0, 0, 0, 1552, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0] payload_rows_per_dst=[7758, 8192, 8054, 8192, 8166, 5262, 4134, 8190] dup_rows_per_dst=[252, 0, 0, 0, 0, 30, 2262, 5044] combine_rows_per_src=[7758, 8192, 8054, 8192, 8166, 5262, 4134, 8190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57344 payload_rows_remote=51354 dup_rows=8192 dup_rows_remote=8192 dedup_ratio=0.125000 router_tokens_per_expert=[0, 2, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2995, 0, 0, 0, 0, 0, 0, 0, 441, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 5990, 2202, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 8192] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 5990, 2202, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57046 payload_rows_remote=48854 dup_rows=8490 dup_rows_remote=354 dedup_ratio=0.129547 router_tokens_per_expert=[0, 83, 3707, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2587, 177, 0, 84, 0, 0, 0, 0, 741, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 0] payload_rows_per_dst=[7442, 8192, 8098, 8192, 8192, 5634, 3104, 8192] dup_rows_per_dst=[166, 0, 0, 0, 0, 62, 126, 8136] combine_rows_per_src=[7442, 8192, 8098, 8192, 8192, 5634, 3104, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=213614592 dispatch_meta_bytes=229296 combine_payload_bytes=213614592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213843888 ingress_bytes=238780416 link_bytes=238780416 token_bytes=427458480 weight_movement_bytes=25165824 total_nvlink_bytes=452624304 weight_to_token_ratio=0.058873 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.816843070e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=194977792 dispatch_meta_bytes=229376 combine_payload_bytes=194977792 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=195207168 ingress_bytes=245309440 link_bytes=245309440 token_bytes=390184960 weight_movement_bytes=50331648 total_nvlink_bytes=440516608 weight_to_token_ratio=0.128994 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.866521399e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=203907072 dispatch_meta_bytes=229480 combine_payload_bytes=203907072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=204136552 ingress_bytes=229072896 link_bytes=229072896 token_bytes=408043624 weight_movement_bytes=25165824 total_nvlink_bytes=433209448 weight_to_token_ratio=0.061674 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.742980059e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=57270 payload_rows_remote=52016 dup_rows=8266 dup_rows_remote=8256 dedup_ratio=0.126129 router_tokens_per_expert=[0, 5, 4008, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 1264, 283, 0, 29, 0, 0, 0, 0, 1823, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0] payload_rows_per_dst=[8016, 8192, 8126, 8192, 8192, 3106, 5254, 8192] dup_rows_per_dst=[10, 0, 0, 0, 10, 46, 10, 8190] combine_rows_per_src=[8016, 8192, 8126, 8192, 8192, 3106, 5254, 8192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=210345984 dispatch_meta_bytes=238184 combine_payload_bytes=210345984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=210584168 ingress_bytes=235511808 link_bytes=235511808 token_bytes=420930152 weight_movement_bytes=25165824 total_nvlink_bytes=446095976 weight_to_token_ratio=0.059786 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791972740e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6412997536 prefetch_bytes=402653184 reduce_bytes=427819008 weight_movement_bytes=830472192 egress_bytes=3208336288 ingress_bytes=4035133440 link_bytes=4035133440 total_nvlink_bytes=7243469728 weight_to_token_ratio=0.129498 prefetch_share_of_ingress=0.099787 dedup_ratio=0.146286 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=17 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=15748 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.918918946e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=200105984 dispatch_meta_bytes=196832 combine_payload_bytes=200105984 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=200302816 ingress_bytes=351100928 link_bytes=351100928 token_bytes=400408800 weight_movement_bytes=150994944 total_nvlink_bytes=551403744 weight_to_token_ratio=0.377102 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18048 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.671472387e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6018830376 prefetch_bytes=427819008 reduce_bytes=402653184 weight_movement_bytes=830472192 egress_bytes=3011186728 ingress_bytes=3838115840 link_bytes=3838115840 total_nvlink_bytes=6849302568 weight_to_token_ratio=0.137979 prefetch_share_of_ingress=0.111466 dedup_ratio=0.184032 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=16 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=14859 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.825226678e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56856 payload_rows_remote=48788 dup_rows=8680 dup_rows_remote=8680 dedup_ratio=0.132446 router_tokens_per_expert=[0, 18, 3908, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2025, 228, 0, 10, 0, 0, 0, 0, 1366, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0] payload_rows_per_dst=[7828, 8192, 8068, 8192, 8192, 8192, 8192, 0] dup_rows_per_dst=[24, 0, 0, 0, 0, 4522, 4134, 0] combine_rows_per_src=[7828, 8192, 8068, 8192, 8192, 8192, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=56498 payload_rows_remote=48306 dup_rows=9038 dup_rows_remote=6516 dedup_ratio=0.137909 router_tokens_per_expert=[0, 208, 4035, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 3, 0, 0, 3584, 40, 0, 12, 0, 0, 0, 0, 29, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] payload_rows_per_dst=[8192, 8192, 8178, 8172, 8168, 7404, 8192, 0] dup_rows_per_dst=[2522, 62, 556, 0, 84, 5150, 664, 0] combine_rows_per_src=[8192, 8192, 8178, 8172, 8168, 7404, 8192, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6311598520 prefetch_bytes=402653184 reduce_bytes=327155712 weight_movement_bytes=729808896 egress_bytes=3157629368 ingress_bytes=3883778048 link_bytes=3883778048 total_nvlink_bytes=7041407416 weight_to_token_ratio=0.115630 prefetch_share_of_ingress=0.103676 dedup_ratio=0.154247 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=13 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=15494 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.846941468e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=213057536 dispatch_meta_bytes=241088 combine_payload_bytes=213057536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=213298624 ingress_bytes=238223360 link_bytes=238223360 token_bytes=426356160 weight_movement_bytes=25165824 total_nvlink_bytes=451521984 weight_to_token_ratio=0.059025 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=18048 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.812604518e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6437139752 prefetch_bytes=377487360 reduce_bytes=452984832 weight_movement_bytes=830472192 egress_bytes=3220428072 ingress_bytes=4047183872 link_bytes=4047183872 total_nvlink_bytes=7267611944 weight_to_token_ratio=0.129013 prefetch_share_of_ingress=0.093272 dedup_ratio=0.150272 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=18 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=15113 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.924649563e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5764693808 prefetch_bytes=402653184 reduce_bytes=478150656 weight_movement_bytes=880803840 egress_bytes=2884116272 ingress_bytes=3761381376 link_bytes=3761381376 total_nvlink_bytes=6645497648 weight_to_token_ratio=0.152793 prefetch_share_of_ingress=0.107049 dedup_ratio=0.212236 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=19 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=14732 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.788735390e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=199835648 dispatch_meta_bytes=229872 combine_payload_bytes=199835648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=200065520 ingress_bytes=225001472 link_bytes=225001472 token_bytes=399901168 weight_movement_bytes=25165824 total_nvlink_bytes=425066992 weight_to_token_ratio=0.062930 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.712001227e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=197861376 dispatch_meta_bytes=219288 combine_payload_bytes=197861376 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=198080664 ingress_bytes=223027200 link_bytes=223027200 token_bytes=395942040 weight_movement_bytes=25165824 total_nvlink_bytes=421107864 weight_to_token_ratio=0.063559 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.696979298e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6184482176 prefetch_bytes=427819008 reduce_bytes=377487360 weight_movement_bytes=805306368 egress_bytes=3094050176 ingress_bytes=3895738368 link_bytes=3895738368 total_nvlink_bytes=6989788544 weight_to_token_ratio=0.130214 prefetch_share_of_ingress=0.109817 dedup_ratio=0.168142 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=15 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.044347 gemm_inner_pad_upper_bound=14732 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.852629231e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=6177969832 prefetch_bytes=301989888 reduce_bytes=427819008 weight_movement_bytes=729808896 egress_bytes=3090773672 ingress_bytes=3817005056 link_bytes=3817005056 total_nvlink_bytes=6907778728 weight_to_token_ratio=0.118131 prefetch_share_of_ingress=0.079117 dedup_ratio=0.170755 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=17 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.044347 gemm_inner_pad_upper_bound=14605 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.815187386e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5813834864 prefetch_bytes=427819008 reduce_bytes=276824064 weight_movement_bytes=704643072 egress_bytes=2908648560 ingress_bytes=3609829376 link_bytes=3609829376 total_nvlink_bytes=6518477936 weight_to_token_ratio=0.121201 prefetch_share_of_ingress=0.118515 dedup_ratio=0.203117 prefetch_slots=34 unused_prefetch_slots=222 slot_utilization=0.132812 reduce_slots=11 remote_demand=17 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=15875 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.716664414e-04 prefetch_bytes_per_gemm_flop=2.034505208e-05 anomalies=none + [2026-09-21 08:31:47] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 215.3 | throughput per GPU (TFLOP/s/GPU): 149.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.591659E+00 | loss scale: 1.0 | grad norm: 3.159 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-09-21 08:31:47 +W0921 08:31:48.477000 669414 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 08:31:48.478000 669414 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 670893 closing signal SIGTERM +W0921 08:31:48.478000 669414 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 670894 closing signal SIGTERM +W0921 08:31:48.480000 669414 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 670895 closing signal SIGTERM +W0921 08:31:48.481000 669414 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 670896 closing signal SIGTERM +W0921 08:31:48.482000 669414 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 670897 closing signal SIGTERM +W0921 08:31:48.483000 669414 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 670898 closing signal SIGTERM +W0921 08:31:48.485000 669414 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 670899 closing signal SIGTERM +W0921 08:31:48.486000 669414 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 670900 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 669414 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..c79c93e0f584d5065f7aa2807798f5ef5df25f6b --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/mem_samples.csv @@ -0,0 +1,184 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 4 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 2 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 819 +1, 803 +2, 815 +3, 803 +4, 813 +5, 801 +6, 815 +7, 817 +0, 1079 +1, 1079 +2, 1079 +3, 1079 +4, 1079 +5, 1079 +6, 1079 +7, 1079 +0, 1823 +1, 1823 +2, 1823 +3, 1823 +4, 1823 +5, 1823 +6, 1823 +7, 1823 +0, 13295 +1, 13273 +2, 13291 +3, 13279 +4, 13299 +5, 13293 +6, 13303 +7, 13303 +0, 14301 +1, 14273 +2, 14295 +3, 14295 +4, 14295 +5, 14311 +6, 14349 +7, 14323 +0, 14945 +1, 14935 +2, 14959 +3, 14967 +4, 14979 +5, 14975 +6, 14979 +7, 14951 +0, 15599 +1, 15603 +2, 15603 +3, 15607 +4, 15597 +5, 15605 +6, 15605 +7, 15595 +0, 15877 +1, 15883 +2, 15883 +3, 15885 +4, 15875 +5, 15883 +6, 15883 +7, 15875 +0, 26363 +1, 26341 +2, 26373 +3, 26351 +4, 26369 +5, 26365 +6, 26349 +7, 26403 +0, 26775 +1, 26781 +2, 26801 +3, 26783 +4, 26793 +5, 26781 +6, 26783 +7, 26793 +0, 27067 +1, 27097 +2, 27137 +3, 27079 +4, 27087 +5, 27115 +6, 27091 +7, 27159 +0, 28023 +1, 28041 +2, 28051 +3, 28031 +4, 28059 +5, 28027 +6, 28041 +7, 28061 +0, 35741 +1, 35739 +2, 35727 +3, 35745 +4, 35751 +5, 35743 +6, 35745 +7, 35751 +0, 35741 +1, 35739 +2, 35727 +3, 35745 +4, 35751 +5, 35743 +6, 35745 +7, 35751 +0, 4942 +1, 4942 +2, 4942 +3, 5262 +4, 35367 +5, 6542 +6, 4942 +7, 4942 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..72a1acc3d73bd54aed84e7a6357dc6a791a7aec4 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/run.env @@ -0,0 +1,105 @@ +backend=moonep +config=OLMOE_1B_7B_L16 +family=tierB +source_model=OLMoE-1B-7B-0924 +standard_num_layers=16 +base_rung= +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=a6a1a221 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=16 +hidden=2048 +ffn_hidden=1024 +moe_ffn_hidden=1024 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=6291456 +heads=16 +kv_groups=16 +kv_channels=128 +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +q_lora_rank= +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=0 +moe_layers=16 +rotary_base=10000 +rotary_percent=1.0 +rotary_scaling= +mscale= +mscale_all_dim= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +router_groups= +router_group_topk= +router_score_function=softmax +router_expert_bias=0 +router_bias_update_rate= +router_topk_scaling_factor= +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ac277c8b1f70457ddb9b37bf1f087ae16263b479 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/train.log @@ -0,0 +1,5034 @@ +[08:31:52] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + OLMOE_1B_7B_L16 / moonep (slots/rank=8 policy=matched) + E=64 K=8 H'=1024 m=8 + L=16 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ a6a1a221 +============================================================== +W0921 08:32:02.336000 677271 torch/distributed/run.py:766] +W0921 08:32:02.336000 677271 torch/distributed/run.py:766] ***************************************** +W0921 08:32:02.336000 677271 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 08:32:02.336000 677271 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 1024 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1024 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 16 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 73 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 46 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 2347 is connected to 5 is connected to is connected to 7 is connected to 77 peer ranks. is connected to 7 peer ranks. peer ranks. Expected number of connected peer ranks is : 7 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7 peer ranks. Expected number of connected peer ranks is : 77 + +Expected number of connected peer ranks is : 7 + +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 000 is connected to is connected to is connected to 000 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 000 + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W921 08:32:19.175117140 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W921 08:32:19.185010441 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 08:32:19.185116068 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 08:32:19.185127492 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 08:32:19.185153987 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W921 08:32:19.185168183 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 08:32:19.185199047 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 08:32:19.354154275 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.810 seconds +time to initialize megatron (seconds): 7.660 +[after megatron is initialized] datetime: 2026-09-21 08:32:25 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 1281951744 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 7 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (46542848 elements, 46542848 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.pre_mlp_layernorm.weight +Params for bucket 3 (50737152 elements, 50737152 padded size): + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 4 (50737152 elements, 50737152 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 5 (50737152 elements, 50737152 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight +Params for bucket 6 (50737152 elements, 50737152 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 7 (124131328 elements, 124131328 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 2 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 +Params for bucket 3 (41943040 elements, 41943040 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (41943040 elements, 41943040 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 5 (41943040 elements, 41943040 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 6 (41943040 elements, 41943040 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (41943040 elements, 41943040 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 8 (41943040 elements, 41943040 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (41943040 elements, 41943040 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (41943040 elements, 41943040 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 +Params for bucket 11 (41943040 elements, 41943040 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 12 (41943040 elements, 41943040 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 13 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 14 (41943040 elements, 41943040 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 15 (41943040 elements, 41943040 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 16 (41943040 elements, 41943040 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 17 (41943040 elements, 41943040 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 +Params for bucket 18 (41943040 elements, 41943040 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (41943040 elements, 41943040 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 20 (8388608 elements, 8388608 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 08:32:28 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 08:32:28 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3087.73, 3088.26) + train/valid/test-data-iterators-setup ..........: (65.04, 67.93) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-09-21 08:32:28 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=2048, E=64, K=8, NvS=34800, NvS_padded=34816 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 136.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.26 MiB + total_size : 139.26 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[28, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [48, -1, -1, -1, -1, -1, -1, -1], [5, -1, -1, -1, -1, -1, -1, -1], [28, -1, -1, -1, -1, -1, -1, -1], [30, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [4096, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[57, -1, -1, -1, -1, -1, -1, -1], [59, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1], [14, -1, -1, -1, -1, -1, -1, -1], [26, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[60, -1, -1, -1, -1, -1, -1, -1], [44, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1], [7, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1], [2, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [25, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-09-21 08:32:48] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 19962.5 | throughput per GPU (TFLOP/s/GPU): 1.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.118884E+01 | loss scale: 1.0 | grad norm: 34.559 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 6.71 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 6.92 +Number of parameters in most loaded shard in billions: 6.9171 +Theoretical memory footprints: weight and optimizer=49474.72 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 16627.5546875 | max allocated: 16999.91845703125 | reserved: 17484.0 | max reserved: 17484.0 + [2026-09-21 08:32:48] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 249.8 | throughput per GPU (TFLOP/s/GPU): 129.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.121123E+01 | loss scale: 1.0 | grad norm: 35.606 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38914 dup_rows=21142 dup_rows_remote=18644 dedup_ratio=0.322601 router_tokens_per_expert=[518, 541, 520, 558, 512, 532, 536, 531, 441, 525, 529, 543, 528, 495, 539, 543, 515, 539, 543, 487, 519, 504, 517, 507, 522, 528, 503, 504, 511, 494, 512, 509, 500, 512, 537, 494, 517, 536, 531, 517, 495, 503, 473, 523, 508, 521, 502, 431, 515, 488, 485, 506, 495, 522, 538, 525, 517, 515, 481, 519, 541, 488, 477, 451] payload_rows_per_dst=[5564, 5518, 5494, 5562, 5636, 5496, 5644, 5480] dup_rows_per_dst=[2708, 2658, 2560, 2656, 2808, 2564, 2690, 2498] combine_rows_per_src=[5564, 5518, 5494, 5562, 5636, 5496, 5644, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44582 payload_rows_remote=38930 dup_rows=20954 dup_rows_remote=18168 dedup_ratio=0.319733 router_tokens_per_expert=[522, 528, 529, 533, 525, 515, 490, 528, 544, 551, 529, 525, 536, 516, 499, 519, 502, 512, 510, 550, 519, 513, 532, 553, 532, 500, 522, 477, 478, 493, 507, 542, 531, 496, 511, 505, 513, 486, 488, 536, 490, 488, 515, 515, 545, 478, 495, 532, 536, 525, 496, 488, 479, 507, 505, 530, 476, 479, 475, 493, 468, 505, 527, 524] payload_rows_per_dst=[5688, 5652, 5684, 5534, 5524, 5528, 5518, 5454] dup_rows_per_dst=[2652, 2786, 2698, 2568, 2608, 2588, 2614, 2440] combine_rows_per_src=[5688, 5652, 5684, 5534, 5524, 5528, 5518, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=39064 dup_rows=20946 dup_rows_remote=18444 dedup_ratio=0.319611 router_tokens_per_expert=[538, 520, 480, 521, 536, 519, 473, 501, 512, 504, 505, 520, 576, 521, 501, 502, 480, 523, 533, 492, 517, 499, 540, 499, 522, 531, 474, 538, 487, 494, 499, 487, 536, 501, 544, 467, 527, 523, 555, 502, 484, 483, 515, 525, 559, 514, 492, 494, 512, 468, 510, 512, 511, 468, 515, 518, 558, 504, 524, 522, 529, 509, 548, 495] payload_rows_per_dst=[5540, 5588, 5554, 5554, 5672, 5544, 5526, 5612] dup_rows_per_dst=[2636, 2694, 2612, 2510, 2638, 2588, 2502, 2766] combine_rows_per_src=[5540, 5588, 5554, 5554, 5672, 5544, 5526, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38870 dup_rows=21082 dup_rows_remote=18378 dedup_ratio=0.321686 router_tokens_per_expert=[506, 510, 520, 493, 496, 531, 502, 532, 493, 490, 541, 536, 525, 486, 476, 514, 526, 507, 509, 525, 497, 543, 501, 529, 515, 546, 498, 515, 516, 513, 509, 532, 518, 503, 493, 536, 515, 498, 507, 514, 535, 527, 526, 534, 514, 538, 506, 530, 504, 486, 523, 497, 482, 497, 501, 486, 538, 502, 530, 536, 482, 508, 481, 489] payload_rows_per_dst=[5570, 5490, 5582, 5584, 5516, 5688, 5452, 5572] dup_rows_per_dst=[2610, 2632, 2692, 2704, 2652, 2732, 2500, 2560] combine_rows_per_src=[5570, 5490, 5582, 5584, 5516, 5688, 5452, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44242 payload_rows_remote=38706 dup_rows=21294 dup_rows_remote=18672 dedup_ratio=0.324921 router_tokens_per_expert=[533, 527, 524, 532, 487, 532, 477, 534, 543, 489, 504, 510, 483, 493, 495, 502, 516, 478, 524, 481, 521, 533, 531, 495, 504, 516, 517, 493, 495, 523, 498, 557, 499, 499, 506, 524, 495, 502, 528, 509, 492, 549, 496, 502, 545, 508, 462, 544, 495, 525, 485, 529, 520, 517, 527, 512, 506, 541, 518, 515, 510, 541, 511, 509] payload_rows_per_dst=[5496, 5482, 5536, 5564, 5538, 5508, 5508, 5610] dup_rows_per_dst=[2796, 2556, 2622, 2642, 2586, 2688, 2712, 2692] combine_rows_per_src=[5496, 5482, 5536, 5564, 5538, 5508, 5508, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44360 payload_rows_remote=38772 dup_rows=21176 dup_rows_remote=18564 dedup_ratio=0.323120 router_tokens_per_expert=[534, 513, 505, 523, 489, 506, 507, 516, 504, 504, 526, 500, 523, 516, 506, 531, 492, 504, 480, 541, 505, 506, 541, 494, 509, 495, 515, 527, 523, 507, 492, 542, 508, 502, 469, 492, 478, 512, 509, 488, 506, 542, 533, 501, 493, 520, 530, 535, 509, 501, 520, 506, 543, 499, 516, 480, 531, 505, 541, 550, 513, 522, 532, 506] payload_rows_per_dst=[5588, 5530, 5478, 5560, 5454, 5562, 5580, 5608] dup_rows_per_dst=[2612, 2690, 2654, 2660, 2462, 2758, 2568, 2772] combine_rows_per_src=[5588, 5530, 5478, 5560, 5454, 5562, 5580, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44506 payload_rows_remote=38900 dup_rows=21030 dup_rows_remote=18346 dedup_ratio=0.320892 router_tokens_per_expert=[494, 504, 535, 521, 540, 482, 486, 489, 521, 511, 527, 483, 496, 463, 520, 523, 526, 498, 534, 479, 515, 510, 488, 489, 545, 506, 501, 490, 515, 556, 507, 496, 527, 537, 500, 495, 500, 537, 478, 571, 498, 498, 532, 542, 485, 481, 500, 497, 532, 550, 506, 509, 531, 501, 532, 523, 543, 530, 522, 514, 478, 533, 526, 510] payload_rows_per_dst=[5532, 5538, 5520, 5610, 5606, 5486, 5590, 5624] dup_rows_per_dst=[2570, 2550, 2558, 2622, 2684, 2580, 2778, 2688] combine_rows_per_src=[5532, 5538, 5520, 5610, 5606, 5486, 5590, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159391744 dispatch_meta_bytes=230232 combine_payload_bytes=159391744 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159621976 ingress_bytes=209723392 link_bytes=209723392 token_bytes=319013720 weight_movement_bytes=50331648 total_nvlink_bytes=369345368 weight_to_token_ratio=0.157773 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595752691e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159457280 dispatch_meta_bytes=228392 combine_payload_bytes=159457280 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159685672 ingress_bytes=209788928 link_bytes=209788928 token_bytes=319142952 weight_movement_bytes=50331648 total_nvlink_bytes=369474600 weight_to_token_ratio=0.157709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596251344e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44520 payload_rows_remote=38910 dup_rows=21016 dup_rows_remote=18400 dedup_ratio=0.320679 router_tokens_per_expert=[505, 492, 488, 494, 482, 509, 531, 486, 517, 507, 537, 501, 492, 480, 514, 538, 526, 566, 530, 527, 491, 548, 470, 488, 499, 517, 481, 518, 563, 531, 487, 507, 552, 552, 477, 490, 524, 511, 486, 484, 503, 491, 511, 515, 531, 502, 523, 537, 516, 509, 545, 522, 499, 527, 522, 537, 503, 525, 517, 529, 492, 519, 474, 521] payload_rows_per_dst=[5380, 5588, 5648, 5596, 5544, 5610, 5666, 5488] dup_rows_per_dst=[2594, 2584, 2644, 2610, 2608, 2616, 2688, 2672] combine_rows_per_src=[5380, 5588, 5648, 5596, 5544, 5610, 5666, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158810112 dispatch_meta_bytes=229344 combine_payload_bytes=158810112 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159039456 ingress_bytes=234307584 link_bytes=234307584 token_bytes=317849568 weight_movement_bytes=75497472 total_nvlink_bytes=393347040 weight_to_token_ratio=0.237526 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782809987e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159211520 dispatch_meta_bytes=228992 combine_payload_bytes=159211520 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159440512 ingress_bytes=184377344 link_bytes=184377344 token_bytes=318652032 weight_movement_bytes=25165824 total_nvlink_bytes=343817856 weight_to_token_ratio=0.078976 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402898552e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160006144 dispatch_meta_bytes=230032 combine_payload_bytes=160006144 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160236176 ingress_bytes=185171968 link_bytes=185171968 token_bytes=320242320 weight_movement_bytes=25165824 total_nvlink_bytes=345408144 weight_to_token_ratio=0.078584 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408944722e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158539776 dispatch_meta_bytes=229512 combine_payload_bytes=158539776 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158769288 ingress_bytes=234037248 link_bytes=234037248 token_bytes=317309064 weight_movement_bytes=75497472 total_nvlink_bytes=392806536 weight_to_token_ratio=0.237930 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.780753042e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=228984 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159563384 ingress_bytes=184500224 link_bytes=184500224 token_bytes=318897784 weight_movement_bytes=25165824 total_nvlink_bytes=344063608 weight_to_token_ratio=0.078915 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403833526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44350 payload_rows_remote=38760 dup_rows=21186 dup_rows_remote=18374 dedup_ratio=0.323273 router_tokens_per_expert=[509, 530, 482, 538, 499, 474, 533, 518, 496, 513, 523, 505, 510, 503, 509, 501, 494, 491, 520, 501, 529, 544, 526, 522, 480, 524, 486, 502, 459, 517, 481, 484, 527, 511, 519, 523, 504, 546, 501, 510, 556, 483, 532, 508, 556, 475, 533, 466, 519, 543, 527, 532, 470, 538, 531, 485, 514, 530, 521, 536, 513, 512, 506, 538] payload_rows_per_dst=[5382, 5452, 5830, 5458, 5532, 5550, 5556, 5590] dup_rows_per_dst=[2250, 2606, 2958, 2408, 2750, 2668, 2734, 2812] combine_rows_per_src=[5382, 5452, 5830, 5458, 5532, 5550, 5556, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159375360 dispatch_meta_bytes=229240 combine_payload_bytes=159375360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159604600 ingress_bytes=184541184 link_bytes=184541184 token_bytes=318979960 weight_movement_bytes=25165824 total_nvlink_bytes=344145784 weight_to_token_ratio=0.078895 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404145185e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38826 dup_rows=21146 dup_rows_remote=18470 dedup_ratio=0.322662 router_tokens_per_expert=[554, 501, 495, 517, 516, 511, 547, 489, 509, 480, 536, 554, 541, 484, 509, 507, 501, 507, 498, 542, 509, 523, 487, 489, 509, 536, 501, 525, 485, 487, 483, 490, 504, 524, 516, 553, 462, 491, 492, 488, 514, 512, 509, 513, 472, 493, 521, 557, 522, 542, 515, 512, 509, 529, 548, 497, 551, 511, 496, 493, 529, 513, 543, 515] payload_rows_per_dst=[5538, 5564, 5510, 5454, 5520, 5568, 5624, 5612] dup_rows_per_dst=[2722, 2676, 2602, 2578, 2540, 2614, 2724, 2690] combine_rows_per_src=[5538, 5564, 5510, 5454, 5520, 5568, 5624, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44476 payload_rows_remote=38856 dup_rows=21060 dup_rows_remote=18178 dedup_ratio=0.321350 router_tokens_per_expert=[496, 525, 536, 498, 495, 523, 513, 530, 502, 519, 517, 510, 537, 510, 499, 514, 457, 482, 550, 505, 560, 523, 507, 517, 501, 522, 559, 486, 505, 512, 548, 486, 503, 518, 521, 498, 519, 516, 519, 548, 517, 461, 544, 541, 532, 493, 528, 500, 498, 503, 536, 512, 500, 512, 490, 508, 492, 505, 503, 493, 497, 508, 519, 490] payload_rows_per_dst=[5620, 5594, 5546, 5836, 5484, 5538, 5480, 5378] dup_rows_per_dst=[2882, 2626, 2656, 2986, 2372, 2554, 2618, 2366] combine_rows_per_src=[5620, 5594, 5546, 5836, 5484, 5538, 5480, 5378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38884 dup_rows=21052 dup_rows_remote=18540 dedup_ratio=0.321228 router_tokens_per_expert=[504, 551, 559, 513, 515, 504, 495, 488, 547, 524, 514, 510, 471, 494, 517, 551, 538, 472, 468, 526, 556, 476, 506, 489, 530, 479, 511, 550, 489, 470, 500, 527, 475, 491, 519, 504, 532, 506, 503, 537, 508, 534, 524, 520, 522, 532, 460, 503, 534, 522, 512, 517, 534, 490, 532, 492, 532, 494, 512, 492, 512, 518, 550, 511] payload_rows_per_dst=[5598, 5578, 5516, 5600, 5468, 5552, 5596, 5576] dup_rows_per_dst=[2660, 2678, 2546, 2512, 2666, 2654, 2670, 2666] combine_rows_per_src=[5598, 5578, 5516, 5600, 5468, 5552, 5596, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44222 payload_rows_remote=38738 dup_rows=21314 dup_rows_remote=18788 dedup_ratio=0.325226 router_tokens_per_expert=[519, 531, 538, 512, 524, 535, 503, 535, 486, 530, 506, 485, 549, 566, 538, 511, 493, 476, 502, 484, 471, 533, 468, 534, 497, 517, 496, 493, 549, 522, 509, 506, 504, 477, 497, 499, 524, 506, 523, 530, 544, 483, 508, 538, 522, 498, 484, 572, 476, 531, 532, 505, 487, 460, 521, 493, 526, 500, 477, 533, 480, 553, 525, 542] payload_rows_per_dst=[5656, 5584, 5456, 5500, 5480, 5542, 5484, 5520] dup_rows_per_dst=[2738, 2758, 2466, 2678, 2640, 2756, 2526, 2752] combine_rows_per_src=[5656, 5584, 5456, 5500, 5480, 5542, 5484, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=228536 combine_payload_bytes=158760960 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158989496 ingress_bytes=209092608 link_bytes=209092608 token_bytes=317750456 weight_movement_bytes=50331648 total_nvlink_bytes=368082104 weight_to_token_ratio=0.158400 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590953154e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44400 payload_rows_remote=39004 dup_rows=21136 dup_rows_remote=18464 dedup_ratio=0.322510 router_tokens_per_expert=[486, 498, 502, 517, 511, 518, 500, 512, 520, 494, 501, 502, 510, 509, 487, 486, 543, 471, 460, 516, 514, 523, 498, 509, 511, 499, 516, 517, 502, 543, 524, 498, 526, 519, 504, 543, 503, 516, 526, 521, 527, 516, 551, 567, 529, 495, 467, 549, 515, 503, 529, 496, 534, 541, 488, 535, 522, 499, 526, 476, 515, 522, 515, 496] payload_rows_per_dst=[5506, 5506, 5396, 5558, 5608, 5612, 5672, 5542] dup_rows_per_dst=[2582, 2512, 2672, 2662, 2708, 2790, 2610, 2600] combine_rows_per_src=[5506, 5506, 5396, 5558, 5608, 5612, 5672, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159031296 dispatch_meta_bytes=229184 combine_payload_bytes=159031296 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159260480 ingress_bytes=209362944 link_bytes=209362944 token_bytes=318291776 weight_movement_bytes=50331648 total_nvlink_bytes=368623424 weight_to_token_ratio=0.158131 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593010098e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=228136 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159382312 ingress_bytes=209485824 link_bytes=209485824 token_bytes=318536488 weight_movement_bytes=50331648 total_nvlink_bytes=368868136 weight_to_token_ratio=0.158009 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593945073e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=229696 combine_payload_bytes=159268864 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159498560 ingress_bytes=184434688 link_bytes=184434688 token_bytes=318767424 weight_movement_bytes=25165824 total_nvlink_bytes=343933248 weight_to_token_ratio=0.078947 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403334873e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44582 payload_rows_remote=38888 dup_rows=20954 dup_rows_remote=18246 dedup_ratio=0.319733 router_tokens_per_expert=[520, 522, 478, 496, 499, 480, 517, 488, 539, 485, 495, 512, 544, 508, 498, 541, 526, 505, 486, 502, 497, 501, 499, 551, 499, 522, 543, 518, 520, 494, 517, 471, 517, 500, 545, 534, 513, 551, 521, 520, 530, 476, 531, 474, 486, 508, 511, 518, 505, 500, 531, 494, 513, 495, 539, 508, 528, 506, 488, 548, 527, 537, 523, 518] payload_rows_per_dst=[5482, 5634, 5584, 5556, 5694, 5504, 5534, 5594] dup_rows_per_dst=[2518, 2610, 2550, 2612, 2708, 2564, 2636, 2756] combine_rows_per_src=[5482, 5634, 5584, 5556, 5694, 5504, 5534, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158670848 dispatch_meta_bytes=230104 combine_payload_bytes=158670848 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158900952 ingress_bytes=209002496 link_bytes=209002496 token_bytes=317571800 weight_movement_bytes=50331648 total_nvlink_bytes=367903448 weight_to_token_ratio=0.158489 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590267506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159760384 dispatch_meta_bytes=229872 combine_payload_bytes=159760384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159990256 ingress_bytes=184926208 link_bytes=184926208 token_bytes=319750640 weight_movement_bytes=25165824 total_nvlink_bytes=344916464 weight_to_token_ratio=0.078705 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407074772e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44528 payload_rows_remote=39050 dup_rows=21008 dup_rows_remote=18416 dedup_ratio=0.320557 router_tokens_per_expert=[534, 529, 526, 535, 538, 522, 519, 498, 499, 492, 525, 505, 499, 527, 511, 521, 504, 544, 528, 532, 496, 487, 533, 500, 483, 499, 522, 485, 469, 531, 526, 554, 532, 505, 552, 513, 545, 521, 482, 533, 516, 507, 508, 497, 473, 513, 504, 517, 522, 490, 483, 522, 486, 464, 506, 563, 519, 521, 504, 478, 498, 475, 541, 505] payload_rows_per_dst=[5652, 5574, 5566, 5528, 5634, 5478, 5532, 5564] dup_rows_per_dst=[2750, 2584, 2682, 2610, 2732, 2592, 2540, 2518] combine_rows_per_src=[5652, 5574, 5566, 5528, 5634, 5478, 5532, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38998 dup_rows=21120 dup_rows_remote=18586 dedup_ratio=0.322266 router_tokens_per_expert=[474, 507, 536, 554, 520, 513, 477, 538, 517, 495, 535, 500, 511, 484, 516, 503, 488, 478, 505, 551, 507, 525, 510, 517, 480, 548, 515, 512, 544, 488, 562, 510, 555, 512, 523, 492, 524, 504, 542, 524, 495, 530, 521, 513, 491, 520, 513, 495, 508, 527, 529, 508, 520, 517, 491, 518, 524, 505, 458, 492, 498, 481, 526, 492] payload_rows_per_dst=[5372, 5482, 5572, 5640, 5600, 5584, 5748, 5418] dup_rows_per_dst=[2422, 2510, 2590, 2746, 2814, 2572, 2932, 2534] combine_rows_per_src=[5372, 5482, 5572, 5640, 5600, 5584, 5748, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=228536 combine_payload_bytes=159285248 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159513784 ingress_bytes=209616896 link_bytes=209616896 token_bytes=318799032 weight_movement_bytes=50331648 total_nvlink_bytes=369130680 weight_to_token_ratio=0.157879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594942380e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=38960 dup_rows=20910 dup_rows_remote=18172 dedup_ratio=0.319061 router_tokens_per_expert=[476, 503, 562, 488, 519, 493, 519, 549, 516, 557, 515, 556, 503, 498, 517, 540, 503, 517, 505, 503, 536, 447, 483, 506, 546, 485, 500, 508, 509, 514, 484, 468, 501, 527, 518, 520, 551, 495, 544, 559, 508, 525, 504, 529, 531, 474, 488, 510, 479, 504, 527, 553, 518, 497, 506, 478, 510, 508, 490, 517, 516, 508, 506, 542] payload_rows_per_dst=[5576, 5666, 5538, 5518, 5650, 5586, 5556, 5536] dup_rows_per_dst=[2642, 2738, 2462, 2510, 2780, 2552, 2568, 2658] combine_rows_per_src=[5576, 5666, 5538, 5518, 5650, 5586, 5556, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44280 payload_rows_remote=38692 dup_rows=21256 dup_rows_remote=18560 dedup_ratio=0.324341 router_tokens_per_expert=[536, 524, 488, 530, 505, 531, 495, 473, 513, 523, 527, 506, 471, 557, 550, 529, 498, 490, 509, 513, 525, 540, 488, 520, 499, 498, 529, 514, 494, 501, 525, 532, 505, 535, 518, 479, 532, 495, 530, 471, 525, 504, 503, 524, 527, 489, 510, 511, 500, 485, 503, 494, 509, 471, 521, 503, 509, 551, 535, 483, 511, 522, 556, 524] payload_rows_per_dst=[5588, 5612, 5714, 5578, 5306, 5544, 5442, 5496] dup_rows_per_dst=[2696, 2624, 2990, 2606, 2286, 2656, 2530, 2868] combine_rows_per_src=[5588, 5612, 5714, 5578, 5306, 5544, 5442, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159948800 dispatch_meta_bytes=229864 combine_payload_bytes=159948800 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160178664 ingress_bytes=210280448 link_bytes=210280448 token_bytes=320127464 weight_movement_bytes=50331648 total_nvlink_bytes=370459112 weight_to_token_ratio=0.157224 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599991244e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159735808 dispatch_meta_bytes=230336 combine_payload_bytes=159735808 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159966144 ingress_bytes=210067456 link_bytes=210067456 token_bytes=319701952 weight_movement_bytes=50331648 total_nvlink_bytes=370033600 weight_to_token_ratio=0.157433 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598370621e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=38972 dup_rows=20988 dup_rows_remote=18390 dedup_ratio=0.320251 router_tokens_per_expert=[526, 509, 516, 567, 521, 521, 493, 507, 506, 535, 465, 515, 532, 517, 520, 525, 474, 532, 520, 457, 501, 533, 514, 448, 500, 515, 532, 474, 529, 536, 497, 504, 533, 463, 552, 523, 505, 494, 492, 508, 491, 541, 503, 507, 501, 522, 495, 496, 498, 538, 558, 519, 520, 494, 521, 490, 495, 487, 545, 522, 524, 521, 514, 555] payload_rows_per_dst=[5592, 5612, 5456, 5576, 5604, 5516, 5598, 5594] dup_rows_per_dst=[2728, 2618, 2502, 2598, 2536, 2596, 2678, 2732] combine_rows_per_src=[5592, 5612, 5456, 5576, 5604, 5516, 5598, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38858 dup_rows=21058 dup_rows_remote=18346 dedup_ratio=0.321320 router_tokens_per_expert=[545, 529, 549, 552, 496, 527, 505, 541, 494, 557, 510, 514, 514, 526, 486, 511, 504, 535, 527, 539, 496, 484, 506, 508, 541, 519, 508, 475, 494, 491, 507, 510, 555, 543, 514, 527, 478, 502, 468, 509, 508, 482, 524, 481, 480, 489, 523, 473, 523, 543, 524, 516, 522, 510, 506, 522, 482, 486, 508, 502, 494, 537, 534, 503] payload_rows_per_dst=[5648, 5586, 5576, 5518, 5554, 5468, 5620, 5508] dup_rows_per_dst=[2840, 2638, 2622, 2572, 2638, 2452, 2712, 2584] combine_rows_per_src=[5648, 5586, 5576, 5518, 5554, 5468, 5620, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44610 payload_rows_remote=39028 dup_rows=20926 dup_rows_remote=18284 dedup_ratio=0.319305 router_tokens_per_expert=[483, 485, 530, 535, 513, 477, 495, 486, 501, 544, 516, 513, 518, 493, 502, 511, 534, 514, 563, 496, 494, 499, 509, 503, 521, 490, 523, 492, 523, 540, 492, 519, 507, 545, 524, 506, 489, 538, 532, 516, 517, 528, 521, 511, 542, 515, 555, 518, 498, 516, 491, 509, 494, 496, 486, 538, 515, 499, 515, 491, 524, 509, 521, 488] payload_rows_per_dst=[5470, 5574, 5582, 5620, 5614, 5666, 5530, 5554] dup_rows_per_dst=[2538, 2622, 2642, 2580, 2700, 2748, 2526, 2570] combine_rows_per_src=[5470, 5574, 5582, 5620, 5614, 5666, 5530, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159580160 dispatch_meta_bytes=228528 combine_payload_bytes=159580160 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159808688 ingress_bytes=260243456 link_bytes=260243456 token_bytes=319388848 weight_movement_bytes=100663296 total_nvlink_bytes=420052144 weight_to_token_ratio=0.315175 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.980152005e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158482432 dispatch_meta_bytes=229008 combine_payload_bytes=158482432 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158711440 ingress_bytes=208814080 link_bytes=208814080 token_bytes=317193872 weight_movement_bytes=50331648 total_nvlink_bytes=367525520 weight_to_token_ratio=0.158678 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588833877e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=229448 combine_payload_bytes=159629312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159858760 ingress_bytes=184795136 link_bytes=184795136 token_bytes=319488072 weight_movement_bytes=25165824 total_nvlink_bytes=344653896 weight_to_token_ratio=0.078769 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406077466e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159162368 dispatch_meta_bytes=228816 combine_payload_bytes=159162368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159391184 ingress_bytes=184328192 link_bytes=184328192 token_bytes=318553552 weight_movement_bytes=25165824 total_nvlink_bytes=343719376 weight_to_token_ratio=0.079000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402524562e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159858688 dispatch_meta_bytes=229248 combine_payload_bytes=159858688 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160087936 ingress_bytes=185024512 link_bytes=185024512 token_bytes=319946624 weight_movement_bytes=25165824 total_nvlink_bytes=345112448 weight_to_token_ratio=0.078656 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407822752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38884 dup_rows=21118 dup_rows_remote=18446 dedup_ratio=0.322235 router_tokens_per_expert=[512, 509, 539, 542, 519, 512, 543, 519, 510, 481, 527, 529, 479, 510, 541, 505, 477, 508, 510, 509, 534, 488, 513, 479, 497, 501, 516, 489, 537, 500, 493, 491, 524, 514, 521, 504, 531, 534, 472, 503, 511, 507, 506, 504, 507, 533, 550, 550, 516, 511, 503, 503, 493, 544, 514, 475, 498, 502, 544, 498, 532, 523, 531, 491] payload_rows_per_dst=[5648, 5524, 5594, 5448, 5534, 5592, 5556, 5522] dup_rows_per_dst=[2742, 2640, 2442, 2600, 2672, 2744, 2562, 2716] combine_rows_per_src=[5648, 5524, 5594, 5448, 5534, 5592, 5556, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44442 payload_rows_remote=38822 dup_rows=21094 dup_rows_remote=18298 dedup_ratio=0.321869 router_tokens_per_expert=[504, 520, 504, 512, 485, 493, 514, 542, 497, 536, 509, 511, 561, 489, 503, 493, 526, 473, 534, 506, 536, 489, 522, 487, 486, 514, 514, 545, 522, 508, 527, 538, 485, 486, 534, 483, 513, 486, 531, 519, 494, 488, 502, 476, 544, 544, 530, 522, 524, 499, 522, 497, 490, 526, 527, 509, 528, 534, 517, 542, 523, 482, 492, 519] payload_rows_per_dst=[5532, 5568, 5542, 5554, 5502, 5528, 5596, 5620] dup_rows_per_dst=[2502, 2736, 2604, 2612, 2572, 2678, 2594, 2796] combine_rows_per_src=[5532, 5568, 5542, 5554, 5502, 5528, 5596, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44668 payload_rows_remote=39040 dup_rows=20868 dup_rows_remote=18236 dedup_ratio=0.318420 router_tokens_per_expert=[492, 477, 518, 494, 505, 535, 497, 499, 495, 553, 459, 469, 511, 536, 532, 490, 530, 544, 498, 522, 520, 524, 501, 488, 521, 531, 521, 524, 545, 504, 508, 559, 516, 518, 510, 516, 512, 518, 510, 524, 509, 519, 476, 529, 497, 497, 570, 533, 495, 515, 461, 515, 514, 500, 477, 512, 505, 567, 540, 518, 505, 480, 471, 537] payload_rows_per_dst=[5482, 5564, 5614, 5686, 5594, 5628, 5468, 5632] dup_rows_per_dst=[2552, 2526, 2640, 2740, 2654, 2632, 2510, 2614] combine_rows_per_src=[5482, 5564, 5614, 5686, 5594, 5628, 5468, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44714 payload_rows_remote=39108 dup_rows=20822 dup_rows_remote=18296 dedup_ratio=0.317719 router_tokens_per_expert=[488, 512, 518, 562, 540, 518, 504, 523, 539, 515, 488, 490, 492, 512, 503, 527, 546, 491, 503, 500, 526, 509, 533, 546, 535, 526, 552, 531, 494, 565, 527, 475, 529, 499, 528, 466, 527, 511, 493, 508, 539, 485, 504, 484, 478, 499, 512, 532, 476, 488, 493, 499, 527, 527, 519, 519, 479, 503, 492, 488, 513, 502, 509, 550] payload_rows_per_dst=[5654, 5606, 5652, 5642, 5498, 5558, 5538, 5566] dup_rows_per_dst=[2676, 2526, 2656, 2768, 2624, 2508, 2558, 2506] combine_rows_per_src=[5654, 5606, 5652, 5642, 5498, 5558, 5538, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=229320 combine_payload_bytes=159268864 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159498184 ingress_bytes=209600512 link_bytes=209600512 token_bytes=318767048 weight_movement_bytes=50331648 total_nvlink_bytes=369098696 weight_to_token_ratio=0.157895 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594817716e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159014912 dispatch_meta_bytes=228480 combine_payload_bytes=159014912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159243392 ingress_bytes=184180736 link_bytes=184180736 token_bytes=318258304 weight_movement_bytes=25165824 total_nvlink_bytes=343424128 weight_to_token_ratio=0.079074 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401402592e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44476 payload_rows_remote=39010 dup_rows=21060 dup_rows_remote=18414 dedup_ratio=0.321350 router_tokens_per_expert=[529, 518, 483, 470, 528, 505, 523, 500, 546, 514, 536, 495, 525, 523, 513, 519, 485, 507, 466, 516, 515, 545, 498, 557, 523, 492, 488, 526, 512, 513, 523, 509, 543, 497, 490, 553, 471, 518, 516, 513, 517, 483, 508, 514, 522, 506, 494, 522, 510, 501, 488, 489, 514, 547, 475, 525, 503, 547, 538, 480, 499, 533, 532, 518] payload_rows_per_dst=[5466, 5658, 5690, 5596, 5582, 5494, 5384, 5606] dup_rows_per_dst=[2646, 2684, 2768, 2602, 2650, 2608, 2408, 2694] combine_rows_per_src=[5466, 5658, 5690, 5596, 5582, 5494, 5384, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44238 payload_rows_remote=38740 dup_rows=21298 dup_rows_remote=18656 dedup_ratio=0.324982 router_tokens_per_expert=[554, 512, 505, 517, 507, 504, 553, 493, 557, 513, 512, 500, 509, 512, 509, 500, 488, 475, 450, 507, 511, 514, 492, 564, 491, 513, 514, 507, 495, 565, 478, 507, 481, 500, 516, 475, 535, 525, 514, 544, 527, 496, 511, 513, 516, 526, 534, 496, 486, 493, 510, 540, 514, 536, 501, 531, 505, 516, 541, 526, 545, 472, 504, 511] payload_rows_per_dst=[5572, 5528, 5426, 5498, 5524, 5560, 5554, 5576] dup_rows_per_dst=[2718, 2696, 2576, 2642, 2656, 2678, 2668, 2664] combine_rows_per_src=[5572, 5528, 5426, 5498, 5524, 5560, 5554, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159907840 dispatch_meta_bytes=229104 combine_payload_bytes=159907840 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160136944 ingress_bytes=185073664 link_bytes=185073664 token_bytes=320044784 weight_movement_bytes=25165824 total_nvlink_bytes=345210608 weight_to_token_ratio=0.078632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408196742e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44480 payload_rows_remote=38942 dup_rows=21056 dup_rows_remote=18374 dedup_ratio=0.321289 router_tokens_per_expert=[557, 489, 506, 475, 525, 502, 482, 531, 484, 506, 493, 518, 507, 493, 513, 491, 471, 521, 542, 520, 543, 512, 492, 509, 533, 548, 512, 530, 515, 487, 471, 507, 534, 512, 533, 489, 482, 523, 534, 496, 510, 486, 503, 543, 501, 574, 538, 526, 519, 490, 492, 518, 546, 517, 511, 555, 537, 475, 473, 489, 520, 524, 498, 535] payload_rows_per_dst=[5598, 5452, 5538, 5578, 5594, 5644, 5568, 5508] dup_rows_per_dst=[2536, 2558, 2682, 2628, 2612, 2718, 2728, 2594] combine_rows_per_src=[5598, 5452, 5538, 5578, 5594, 5644, 5568, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44410 payload_rows_remote=38754 dup_rows=21126 dup_rows_remote=18490 dedup_ratio=0.322357 router_tokens_per_expert=[539, 536, 490, 496, 531, 509, 513, 531, 482, 474, 515, 509, 520, 549, 516, 523, 507, 470, 504, 517, 504, 479, 522, 542, 505, 543, 485, 533, 478, 565, 527, 493, 500, 503, 495, 472, 513, 502, 539, 495, 492, 471, 531, 492, 548, 550, 517, 533, 487, 557, 519, 523, 503, 525, 519, 513, 490, 500, 484, 532, 526, 541, 491, 498] payload_rows_per_dst=[5638, 5556, 5510, 5530, 5432, 5548, 5656, 5540] dup_rows_per_dst=[2652, 2620, 2580, 2728, 2606, 2720, 2636, 2584] combine_rows_per_src=[5638, 5556, 5510, 5530, 5432, 5548, 5656, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=160186368 dispatch_meta_bytes=229616 combine_payload_bytes=160186368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160415984 ingress_bytes=185352192 link_bytes=185352192 token_bytes=320602352 weight_movement_bytes=25165824 total_nvlink_bytes=345768176 weight_to_token_ratio=0.078495 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410316018e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=229696 combine_payload_bytes=159784960 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160014656 ingress_bytes=235282432 link_bytes=235282432 token_bytes=319799616 weight_movement_bytes=75497472 total_nvlink_bytes=395297088 weight_to_token_ratio=0.236077 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790227454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158679040 dispatch_meta_bytes=229584 combine_payload_bytes=158679040 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158908624 ingress_bytes=209010688 link_bytes=209010688 token_bytes=317587664 weight_movement_bytes=50331648 total_nvlink_bytes=367919312 weight_to_token_ratio=0.158481 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590329837e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=229264 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159735696 ingress_bytes=184672256 link_bytes=184672256 token_bytes=319242128 weight_movement_bytes=25165824 total_nvlink_bytes=344407952 weight_to_token_ratio=0.078830 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405142491e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158736384 dispatch_meta_bytes=228976 combine_payload_bytes=158736384 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158965360 ingress_bytes=234233856 link_bytes=234233856 token_bytes=317701744 weight_movement_bytes=75497472 total_nvlink_bytes=393199216 weight_to_token_ratio=0.237636 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782249002e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44426 payload_rows_remote=38798 dup_rows=21110 dup_rows_remote=18344 dedup_ratio=0.322113 router_tokens_per_expert=[506, 482, 531, 498, 545, 519, 487, 504, 489, 500, 545, 491, 500, 533, 513, 510, 521, 521, 509, 505, 531, 491, 543, 531, 511, 497, 511, 518, 512, 493, 481, 505, 531, 507, 547, 537, 523, 539, 520, 493, 502, 485, 511, 532, 486, 518, 520, 489, 504, 536, 516, 524, 533, 548, 510, 514, 491, 557, 495, 516, 474, 464, 527, 486] payload_rows_per_dst=[5532, 5484, 5592, 5478, 5628, 5602, 5658, 5452] dup_rows_per_dst=[2612, 2678, 2712, 2578, 2766, 2484, 2712, 2568] combine_rows_per_src=[5532, 5484, 5592, 5478, 5628, 5602, 5658, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44604 payload_rows_remote=38918 dup_rows=20932 dup_rows_remote=18248 dedup_ratio=0.319397 router_tokens_per_expert=[509, 499, 509, 496, 520, 507, 519, 489, 483, 503, 502, 477, 521, 550, 523, 543, 535, 503, 535, 465, 490, 528, 483, 505, 518, 521, 514, 513, 536, 487, 518, 515, 523, 526, 532, 482, 506, 492, 499, 524, 516, 537, 527, 551, 513, 506, 514, 522, 464, 550, 491, 510, 470, 533, 497, 504, 524, 513, 533, 497, 549, 506, 529, 512] payload_rows_per_dst=[5486, 5606, 5520, 5620, 5518, 5616, 5552, 5686] dup_rows_per_dst=[2558, 2598, 2480, 2624, 2650, 2764, 2574, 2684] combine_rows_per_src=[5486, 5606, 5520, 5620, 5518, 5616, 5552, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44288 payload_rows_remote=38782 dup_rows=21248 dup_rows_remote=18546 dedup_ratio=0.324219 router_tokens_per_expert=[532, 498, 535, 473, 511, 535, 492, 525, 532, 526, 518, 545, 476, 476, 493, 527, 501, 459, 491, 509, 555, 507, 473, 509, 496, 509, 496, 513, 508, 508, 514, 507, 510, 536, 556, 513, 480, 509, 514, 527, 513, 535, 487, 542, 539, 500, 492, 496, 487, 522, 495, 539, 515, 546, 527, 508, 523, 526, 468, 548, 500, 521, 515, 530] payload_rows_per_dst=[5554, 5526, 5452, 5504, 5564, 5506, 5592, 5590] dup_rows_per_dst=[2648, 2660, 2556, 2598, 2726, 2702, 2686, 2672] combine_rows_per_src=[5554, 5526, 5452, 5504, 5564, 5506, 5592, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=228664 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159636792 ingress_bytes=209739776 link_bytes=209739776 token_bytes=319044920 weight_movement_bytes=50331648 total_nvlink_bytes=369376568 weight_to_token_ratio=0.157757 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595877354e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=38986 dup_rows=21086 dup_rows_remote=18502 dedup_ratio=0.321747 router_tokens_per_expert=[514, 535, 522, 515, 497, 506, 508, 500, 503, 497, 483, 500, 518, 495, 551, 477, 487, 501, 507, 543, 504, 497, 512, 475, 459, 519, 507, 493, 490, 528, 551, 484, 529, 509, 510, 522, 487, 490, 536, 515, 544, 535, 509, 526, 515, 508, 561, 506, 539, 511, 500, 507, 532, 533, 515, 538, 491, 550, 514, 503, 524, 499, 541, 491] payload_rows_per_dst=[5564, 5464, 5546, 5478, 5598, 5660, 5548, 5592] dup_rows_per_dst=[2630, 2584, 2506, 2584, 2598, 2748, 2802, 2634] combine_rows_per_src=[5564, 5464, 5546, 5478, 5598, 5660, 5548, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158916608 dispatch_meta_bytes=228568 combine_payload_bytes=158916608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159145176 ingress_bytes=184082432 link_bytes=184082432 token_bytes=318061784 weight_movement_bytes=25165824 total_nvlink_bytes=343227608 weight_to_token_ratio=0.079122 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400654612e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44610 payload_rows_remote=39034 dup_rows=20926 dup_rows_remote=18442 dedup_ratio=0.319305 router_tokens_per_expert=[536, 522, 527, 495, 522, 543, 516, 529, 484, 531, 499, 508, 529, 519, 522, 498, 503, 505, 530, 527, 555, 527, 482, 507, 535, 486, 530, 525, 489, 462, 480, 523, 535, 530, 476, 497, 511, 474, 534, 508, 531, 474, 488, 521, 511, 469, 513, 528, 548, 486, 484, 521, 501, 518, 519, 494, 509, 527, 541, 528, 497, 517, 513, 519] payload_rows_per_dst=[5620, 5544, 5606, 5576, 5570, 5530, 5564, 5600] dup_rows_per_dst=[2760, 2636, 2666, 2484, 2560, 2540, 2578, 2702] combine_rows_per_src=[5620, 5544, 5606, 5576, 5570, 5530, 5564, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44446 payload_rows_remote=38828 dup_rows=21090 dup_rows_remote=18446 dedup_ratio=0.321808 router_tokens_per_expert=[520, 524, 522, 512, 458, 556, 522, 517, 502, 498, 518, 503, 537, 508, 500, 513, 512, 518, 512, 537, 511, 491, 551, 512, 538, 526, 536, 471, 520, 485, 522, 484, 503, 514, 525, 466, 519, 515, 494, 487, 536, 469, 531, 518, 493, 511, 496, 561, 500, 545, 493, 534, 508, 533, 507, 532, 473, 505, 509, 495, 534, 496, 515, 515] payload_rows_per_dst=[5618, 5622, 5556, 5622, 5626, 5448, 5470, 5484] dup_rows_per_dst=[2644, 2710, 2764, 2774, 2710, 2460, 2602, 2426] combine_rows_per_src=[5618, 5622, 5556, 5622, 5626, 5448, 5470, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=38954 dup_rows=20978 dup_rows_remote=18384 dedup_ratio=0.320099 router_tokens_per_expert=[520, 504, 485, 528, 485, 515, 533, 509, 481, 523, 478, 518, 535, 482, 521, 478, 480, 547, 509, 505, 500, 508, 540, 510, 489, 471, 511, 549, 513, 517, 513, 549, 503, 521, 499, 483, 509, 516, 537, 491, 508, 512, 532, 478, 537, 525, 511, 520, 531, 523, 521, 535, 532, 501, 540, 497, 518, 490, 512, 527, 535, 494, 523, 501] payload_rows_per_dst=[5538, 5504, 5604, 5594, 5552, 5618, 5584, 5564] dup_rows_per_dst=[2620, 2528, 2594, 2630, 2566, 2628, 2776, 2636] combine_rows_per_src=[5538, 5504, 5604, 5594, 5552, 5618, 5584, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158851072 dispatch_meta_bytes=229312 combine_payload_bytes=158851072 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159080384 ingress_bytes=209182720 link_bytes=209182720 token_bytes=317931456 weight_movement_bytes=50331648 total_nvlink_bytes=368263104 weight_to_token_ratio=0.158310 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591638802e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44656 payload_rows_remote=39146 dup_rows=20880 dup_rows_remote=18304 dedup_ratio=0.318604 router_tokens_per_expert=[476, 504, 500, 500, 512, 518, 490, 501, 509, 512, 531, 486, 553, 476, 509, 497, 519, 512, 535, 546, 505, 518, 500, 512, 513, 518, 528, 573, 545, 463, 508, 547, 531, 481, 528, 506, 496, 508, 514, 545, 498, 503, 514, 515, 485, 494, 514, 520, 525, 513, 487, 490, 533, 466, 533, 496, 555, 500, 497, 550, 510, 510, 500, 535] payload_rows_per_dst=[5500, 5546, 5636, 5644, 5620, 5566, 5510, 5634] dup_rows_per_dst=[2502, 2600, 2658, 2746, 2598, 2520, 2576, 2680] combine_rows_per_src=[5500, 5546, 5636, 5644, 5620, 5566, 5510, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159686656 dispatch_meta_bytes=229952 combine_payload_bytes=159686656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159916608 ingress_bytes=184852480 link_bytes=184852480 token_bytes=319603264 weight_movement_bytes=25165824 total_nvlink_bytes=344769088 weight_to_token_ratio=0.078741 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406513788e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=229904 combine_payload_bytes=159883264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160113168 ingress_bytes=185049088 link_bytes=185049088 token_bytes=319996432 weight_movement_bytes=25165824 total_nvlink_bytes=345162256 weight_to_token_ratio=0.078644 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408009747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159039488 dispatch_meta_bytes=229096 combine_payload_bytes=159039488 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159268584 ingress_bytes=209371136 link_bytes=209371136 token_bytes=318308072 weight_movement_bytes=50331648 total_nvlink_bytes=368639720 weight_to_token_ratio=0.158122 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593072430e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=229352 combine_payload_bytes=159555584 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159784936 ingress_bytes=209887232 link_bytes=209887232 token_bytes=319340520 weight_movement_bytes=50331648 total_nvlink_bytes=369672168 weight_to_token_ratio=0.157611 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596999324e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=160342016 dispatch_meta_bytes=229800 combine_payload_bytes=160342016 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160571816 ingress_bytes=210673664 link_bytes=210673664 token_bytes=320913832 weight_movement_bytes=50331648 total_nvlink_bytes=371245480 weight_to_token_ratio=0.156839 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602983163e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44586 payload_rows_remote=38882 dup_rows=20950 dup_rows_remote=18160 dedup_ratio=0.319672 router_tokens_per_expert=[468, 476, 509, 504, 490, 480, 555, 493, 502, 501, 522, 518, 478, 534, 499, 523, 502, 520, 480, 508, 501, 489, 497, 549, 536, 513, 494, 479, 513, 511, 532, 504, 512, 528, 545, 449, 533, 502, 483, 530, 501, 536, 519, 481, 507, 505, 537, 486, 543, 500, 546, 540, 530, 554, 507, 528, 529, 537, 537, 505, 537, 511, 536, 494] payload_rows_per_dst=[5488, 5422, 5554, 5624, 5428, 5660, 5706, 5704] dup_rows_per_dst=[2462, 2452, 2538, 2540, 2638, 2740, 2790, 2790] combine_rows_per_src=[5488, 5422, 5554, 5624, 5428, 5660, 5706, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44552 payload_rows_remote=39082 dup_rows=20984 dup_rows_remote=18440 dedup_ratio=0.320190 router_tokens_per_expert=[518, 501, 492, 530, 546, 489, 512, 511, 517, 548, 508, 525, 526, 518, 535, 534, 488, 517, 558, 478, 521, 506, 497, 524, 492, 477, 491, 516, 561, 470, 504, 505, 495, 545, 510, 499, 512, 484, 476, 486, 514, 534, 506, 509, 534, 502, 520, 532, 526, 513, 510, 469, 507, 493, 515, 521, 492, 536, 491, 531, 518, 521, 519, 533] payload_rows_per_dst=[5560, 5694, 5590, 5494, 5470, 5674, 5510, 5560] dup_rows_per_dst=[2638, 2728, 2588, 2538, 2544, 2628, 2598, 2722] combine_rows_per_src=[5560, 5694, 5590, 5494, 5470, 5674, 5510, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44476 payload_rows_remote=38988 dup_rows=21060 dup_rows_remote=18412 dedup_ratio=0.321350 router_tokens_per_expert=[476, 541, 490, 530, 523, 529, 506, 554, 552, 503, 524, 496, 497, 487, 497, 530, 496, 515, 536, 496, 556, 527, 467, 518, 500, 483, 519, 519, 514, 517, 511, 501, 505, 524, 540, 500, 504, 551, 528, 526, 523, 485, 529, 544, 532, 492, 466, 497, 479, 536, 513, 499, 500, 528, 542, 549, 492, 494, 479, 507, 459, 510, 492, 533] payload_rows_per_dst=[5526, 5568, 5598, 5534, 5670, 5488, 5648, 5444] dup_rows_per_dst=[2772, 2604, 2624, 2594, 2686, 2648, 2644, 2488] combine_rows_per_src=[5526, 5568, 5598, 5534, 5670, 5488, 5648, 5444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159260672 dispatch_meta_bytes=228168 combine_payload_bytes=159260672 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159488840 ingress_bytes=209592320 link_bytes=209592320 token_bytes=318749512 weight_movement_bytes=50331648 total_nvlink_bytes=369081160 weight_to_token_ratio=0.157903 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594755385e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44660 payload_rows_remote=39062 dup_rows=20876 dup_rows_remote=18186 dedup_ratio=0.318542 router_tokens_per_expert=[526, 513, 516, 526, 503, 485, 501, 501, 531, 512, 503, 577, 502, 527, 502, 514, 524, 514, 523, 491, 521, 502, 527, 493, 470, 497, 510, 543, 529, 528, 519, 548, 499, 517, 544, 514, 525, 467, 494, 505, 526, 491, 478, 534, 471, 513, 473, 502, 516, 512, 502, 494, 525, 514, 517, 525, 555, 505, 505, 507, 516, 533, 510, 501] payload_rows_per_dst=[5510, 5550, 5648, 5598, 5568, 5476, 5636, 5674] dup_rows_per_dst=[2632, 2786, 2542, 2690, 2562, 2500, 2574, 2590] combine_rows_per_src=[5510, 5550, 5648, 5598, 5568, 5476, 5636, 5674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44576 payload_rows_remote=39030 dup_rows=20960 dup_rows_remote=18262 dedup_ratio=0.319824 router_tokens_per_expert=[514, 529, 516, 513, 538, 498, 514, 532, 518, 539, 505, 511, 527, 485, 515, 522, 541, 456, 489, 474, 499, 545, 487, 529, 489, 533, 523, 498, 517, 530, 512, 486, 531, 484, 517, 522, 528, 489, 526, 490, 545, 552, 521, 492, 516, 483, 519, 504, 503, 534, 475, 545, 521, 482, 514, 503, 511, 511, 512, 487, 494, 535, 528, 510] payload_rows_per_dst=[5660, 5546, 5518, 5602, 5532, 5608, 5484, 5626] dup_rows_per_dst=[2648, 2698, 2522, 2574, 2642, 2656, 2670, 2550] combine_rows_per_src=[5660, 5546, 5518, 5602, 5532, 5608, 5484, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=160079872 dispatch_meta_bytes=230088 combine_payload_bytes=160079872 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160309960 ingress_bytes=185245696 link_bytes=185245696 token_bytes=320389832 weight_movement_bytes=25165824 total_nvlink_bytes=345555656 weight_to_token_ratio=0.078548 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409505707e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44600 payload_rows_remote=38808 dup_rows=20936 dup_rows_remote=17948 dedup_ratio=0.319458 router_tokens_per_expert=[491, 518, 540, 526, 530, 524, 521, 500, 507, 500, 528, 536, 494, 558, 528, 489, 531, 556, 535, 472, 530, 492, 529, 538, 504, 539, 524, 515, 530, 538, 529, 488, 503, 444, 493, 527, 525, 489, 547, 505, 535, 470, 503, 485, 540, 476, 466, 515, 474, 535, 507, 472, 547, 481, 496, 471, 505, 537, 477, 490, 517, 525, 538, 533] payload_rows_per_dst=[5792, 5618, 5802, 5526, 5458, 5518, 5472, 5414] dup_rows_per_dst=[2988, 2670, 2888, 2478, 2608, 2462, 2492, 2350] combine_rows_per_src=[5792, 5618, 5802, 5526, 5458, 5518, 5472, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44488 payload_rows_remote=38952 dup_rows=21048 dup_rows_remote=18394 dedup_ratio=0.321167 router_tokens_per_expert=[497, 506, 504, 504, 490, 510, 512, 475, 476, 552, 507, 572, 525, 529, 511, 510, 526, 521, 532, 510, 500, 478, 526, 502, 516, 530, 507, 477, 529, 515, 509, 477, 546, 520, 476, 505, 525, 497, 523, 524, 511, 522, 547, 508, 513, 506, 522, 511, 532, 507, 526, 481, 511, 579, 498, 463, 531, 474, 475, 487, 530, 544, 537, 502] payload_rows_per_dst=[5512, 5650, 5536, 5562, 5612, 5546, 5548, 5522] dup_rows_per_dst=[2484, 2714, 2654, 2558, 2620, 2734, 2646, 2638] combine_rows_per_src=[5512, 5650, 5536, 5562, 5612, 5546, 5548, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44384 payload_rows_remote=38748 dup_rows=21152 dup_rows_remote=18454 dedup_ratio=0.322754 router_tokens_per_expert=[531, 484, 506, 514, 499, 514, 481, 517, 512, 514, 512, 522, 500, 492, 538, 511, 500, 522, 469, 468, 509, 504, 536, 526, 525, 532, 520, 513, 498, 540, 546, 483, 484, 509, 518, 468, 540, 510, 538, 480, 513, 500, 490, 511, 507, 498, 511, 548, 511, 557, 474, 534, 532, 498, 549, 512, 526, 502, 486, 523, 513, 526, 535, 527] payload_rows_per_dst=[5450, 5536, 5528, 5582, 5544, 5538, 5636, 5570] dup_rows_per_dst=[2642, 2666, 2540, 2732, 2550, 2618, 2698, 2706] combine_rows_per_src=[5450, 5536, 5528, 5582, 5544, 5538, 5636, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=229600 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159924448 ingress_bytes=235192320 link_bytes=235192320 token_bytes=319619296 weight_movement_bytes=75497472 total_nvlink_bytes=395116768 weight_to_token_ratio=0.236211 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789541805e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159997952 dispatch_meta_bytes=228992 combine_payload_bytes=159997952 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160226944 ingress_bytes=235495424 link_bytes=235495424 token_bytes=320224896 weight_movement_bytes=75497472 total_nvlink_bytes=395722368 weight_to_token_ratio=0.235764 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791848077e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=229168 combine_payload_bytes=159866880 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160096048 ingress_bytes=235364352 link_bytes=235364352 token_bytes=319962928 weight_movement_bytes=75497472 total_nvlink_bytes=395460400 weight_to_token_ratio=0.235957 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790850770e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158957568 dispatch_meta_bytes=227024 combine_payload_bytes=158957568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159184592 ingress_bytes=184123392 link_bytes=184123392 token_bytes=318142160 weight_movement_bytes=25165824 total_nvlink_bytes=343307984 weight_to_token_ratio=0.079102 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400966270e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159547392 dispatch_meta_bytes=229384 combine_payload_bytes=159547392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159776776 ingress_bytes=184713216 link_bytes=184713216 token_bytes=319324168 weight_movement_bytes=25165824 total_nvlink_bytes=344489992 weight_to_token_ratio=0.078810 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405454149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158711808 dispatch_meta_bytes=228808 combine_payload_bytes=158711808 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=158940616 ingress_bytes=183877632 link_bytes=183877632 token_bytes=317652424 weight_movement_bytes=25165824 total_nvlink_bytes=342818248 weight_to_token_ratio=0.079224 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399096321e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44552 payload_rows_remote=38924 dup_rows=20984 dup_rows_remote=18316 dedup_ratio=0.320190 router_tokens_per_expert=[454, 495, 550, 509, 531, 530, 488, 523, 502, 480, 511, 498, 508, 548, 515, 505, 490, 511, 514, 527, 524, 498, 544, 558, 500, 515, 523, 496, 527, 515, 503, 497, 503, 497, 517, 520, 505, 494, 489, 494, 505, 492, 514, 510, 518, 499, 541, 484, 490, 534, 510, 531, 521, 558, 519, 538, 528, 511, 494, 526, 530, 502, 490, 515] payload_rows_per_dst=[5550, 5510, 5390, 5402, 5674, 5682, 5716, 5628] dup_rows_per_dst=[2610, 2560, 2334, 2398, 2906, 2796, 2712, 2668] combine_rows_per_src=[5550, 5510, 5390, 5402, 5674, 5682, 5716, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38788 dup_rows=21068 dup_rows_remote=18368 dedup_ratio=0.321472 router_tokens_per_expert=[539, 507, 518, 511, 490, 491, 506, 493, 511, 507, 490, 490, 543, 504, 514, 545, 494, 518, 493, 503, 449, 515, 514, 527, 554, 554, 478, 486, 509, 486, 486, 518, 491, 482, 532, 544, 552, 531, 509, 549, 511, 575, 525, 511, 493, 534, 521, 513, 503, 506, 475, 468, 538, 503, 537, 488, 522, 523, 520, 502, 562, 514, 491, 500] payload_rows_per_dst=[5530, 5544, 5510, 5534, 5680, 5608, 5492, 5570] dup_rows_per_dst=[2580, 2664, 2516, 2608, 2700, 2758, 2544, 2698] combine_rows_per_src=[5530, 5544, 5510, 5534, 5680, 5608, 5492, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=228960 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159661664 ingress_bytes=184598528 link_bytes=184598528 token_bytes=319094368 weight_movement_bytes=25165824 total_nvlink_bytes=344260192 weight_to_token_ratio=0.078866 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404581506e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44318 payload_rows_remote=38758 dup_rows=21218 dup_rows_remote=18664 dedup_ratio=0.323761 router_tokens_per_expert=[509, 479, 500, 513, 487, 526, 558, 507, 480, 496, 469, 525, 520, 514, 501, 505, 503, 545, 513, 491, 526, 517, 527, 498, 491, 522, 528, 509, 514, 540, 540, 520, 519, 495, 516, 522, 490, 518, 575, 562, 471, 531, 506, 500, 510, 521, 523, 495, 505, 539, 500, 506, 511, 504, 514, 495, 533, 509, 492, 499, 495, 514, 491, 534] payload_rows_per_dst=[5534, 5430, 5614, 5526, 5570, 5560, 5514, 5570] dup_rows_per_dst=[2624, 2590, 2626, 2802, 2824, 2554, 2634, 2564] combine_rows_per_src=[5534, 5430, 5614, 5526, 5570, 5560, 5514, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158875648 dispatch_meta_bytes=228624 combine_payload_bytes=158875648 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159104272 ingress_bytes=209207296 link_bytes=209207296 token_bytes=317979920 weight_movement_bytes=50331648 total_nvlink_bytes=368311568 weight_to_token_ratio=0.158286 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591825797e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44640 payload_rows_remote=39066 dup_rows=20896 dup_rows_remote=18424 dedup_ratio=0.318848 router_tokens_per_expert=[512, 519, 511, 501, 546, 490, 537, 494, 514, 520, 536, 552, 503, 517, 508, 474, 511, 478, 504, 501, 524, 528, 500, 512, 554, 528, 506, 494, 473, 470, 518, 480, 514, 531, 513, 511, 525, 488, 527, 509, 516, 552, 483, 522, 512, 519, 520, 517, 511, 500, 512, 516, 510, 518, 513, 490, 507, 462, 559, 518, 516, 522, 519, 521] payload_rows_per_dst=[5542, 5632, 5498, 5574, 5546, 5598, 5634, 5616] dup_rows_per_dst=[2678, 2616, 2618, 2472, 2690, 2684, 2506, 2632] combine_rows_per_src=[5542, 5632, 5498, 5574, 5546, 5598, 5634, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44208 payload_rows_remote=38662 dup_rows=21328 dup_rows_remote=18672 dedup_ratio=0.325439 router_tokens_per_expert=[495, 528, 498, 527, 510, 525, 529, 508, 476, 487, 525, 500, 551, 528, 511, 523, 521, 509, 512, 542, 492, 511, 561, 511, 509, 525, 532, 510, 517, 514, 535, 494, 495, 482, 517, 509, 541, 498, 524, 523, 498, 475, 524, 503, 478, 488, 519, 506, 546, 533, 474, 512, 550, 495, 481, 531, 478, 523, 497, 512, 516, 488, 540, 496] payload_rows_per_dst=[5512, 5546, 5614, 5538, 5504, 5498, 5546, 5450] dup_rows_per_dst=[2728, 2656, 2704, 2734, 2674, 2484, 2698, 2650] combine_rows_per_src=[5512, 5546, 5614, 5538, 5504, 5498, 5546, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44432 payload_rows_remote=38696 dup_rows=21104 dup_rows_remote=18276 dedup_ratio=0.322021 router_tokens_per_expert=[527, 515, 512, 518, 474, 506, 518, 502, 562, 516, 483, 544, 518, 514, 493, 515, 511, 476, 505, 502, 534, 499, 522, 549, 530, 530, 524, 513, 498, 498, 559, 564, 480, 526, 515, 522, 525, 527, 521, 484, 465, 484, 505, 506, 510, 507, 500, 517, 503, 507, 524, 482, 539, 542, 498, 470, 511, 514, 523, 508, 522, 488, 514, 498] payload_rows_per_dst=[5736, 5626, 5568, 5746, 5584, 5434, 5214, 5524] dup_rows_per_dst=[2828, 2664, 2628, 2872, 2616, 2554, 2310, 2632] combine_rows_per_src=[5736, 5626, 5568, 5746, 5584, 5434, 5214, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44494 payload_rows_remote=38892 dup_rows=21042 dup_rows_remote=18304 dedup_ratio=0.321075 router_tokens_per_expert=[524, 485, 533, 466, 540, 499, 493, 519, 504, 515, 520, 512, 513, 518, 491, 478, 502, 519, 516, 510, 535, 541, 519, 528, 507, 487, 496, 517, 559, 540, 530, 497, 499, 525, 496, 517, 489, 515, 502, 509, 544, 486, 521, 528, 520, 504, 447, 537, 527, 538, 504, 530, 526, 545, 479, 528, 513, 496, 511, 512, 516, 474, 511, 506] payload_rows_per_dst=[5536, 5562, 5602, 5588, 5510, 5524, 5598, 5574] dup_rows_per_dst=[2582, 2540, 2738, 2678, 2594, 2650, 2756, 2504] combine_rows_per_src=[5536, 5562, 5602, 5588, 5510, 5524, 5598, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44530 payload_rows_remote=38824 dup_rows=21006 dup_rows_remote=18192 dedup_ratio=0.320526 router_tokens_per_expert=[494, 537, 510, 491, 465, 490, 487, 510, 531, 503, 550, 530, 527, 514, 475, 526, 507, 517, 563, 521, 559, 438, 528, 513, 523, 518, 486, 494, 494, 530, 517, 522, 515, 511, 507, 485, 517, 540, 499, 504, 535, 497, 502, 505, 486, 489, 479, 512, 515, 498, 560, 535, 526, 571, 527, 528, 498, 517, 490, 503, 519, 504, 521, 503] payload_rows_per_dst=[5502, 5596, 5546, 5604, 5494, 5510, 5706, 5572] dup_rows_per_dst=[2466, 2716, 2746, 2564, 2662, 2500, 2814, 2538] combine_rows_per_src=[5502, 5596, 5546, 5604, 5494, 5510, 5706, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158752768 dispatch_meta_bytes=229688 combine_payload_bytes=158752768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158982456 ingress_bytes=183918592 link_bytes=183918592 token_bytes=317735224 weight_movement_bytes=25165824 total_nvlink_bytes=342901048 weight_to_token_ratio=0.079204 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399407979e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=229960 combine_payload_bytes=160014336 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160244296 ingress_bytes=210345984 link_bytes=210345984 token_bytes=320258632 weight_movement_bytes=50331648 total_nvlink_bytes=370590280 weight_to_token_ratio=0.157159 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600489897e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158359552 dispatch_meta_bytes=229336 combine_payload_bytes=158359552 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158588888 ingress_bytes=208691200 link_bytes=208691200 token_bytes=316948440 weight_movement_bytes=50331648 total_nvlink_bytes=367280088 weight_to_token_ratio=0.158801 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.587898903e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158498816 dispatch_meta_bytes=227888 combine_payload_bytes=158498816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158726704 ingress_bytes=183664640 link_bytes=183664640 token_bytes=317225520 weight_movement_bytes=25165824 total_nvlink_bytes=342391344 weight_to_token_ratio=0.079331 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397475698e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=38968 dup_rows=20978 dup_rows_remote=18408 dedup_ratio=0.320099 router_tokens_per_expert=[511, 527, 530, 517, 493, 457, 517, 526, 475, 488, 521, 561, 523, 495, 515, 542, 521, 504, 484, 524, 506, 512, 524, 488, 525, 525, 572, 501, 520, 513, 542, 455, 511, 487, 473, 509, 510, 529, 502, 512, 505, 523, 523, 503, 519, 496, 530, 499, 504, 484, 530, 516, 517, 533, 524, 535, 505, 486, 500, 492, 511, 556, 515, 515] payload_rows_per_dst=[5496, 5572, 5530, 5564, 5534, 5592, 5680, 5590] dup_rows_per_dst=[2424, 2668, 2596, 2742, 2532, 2604, 2842, 2570] combine_rows_per_src=[5496, 5572, 5530, 5564, 5534, 5592, 5680, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159301632 dispatch_meta_bytes=228784 combine_payload_bytes=159301632 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159530416 ingress_bytes=234799104 link_bytes=234799104 token_bytes=318832048 weight_movement_bytes=75497472 total_nvlink_bytes=394329520 weight_to_token_ratio=0.236794 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786549886e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159023104 dispatch_meta_bytes=228064 combine_payload_bytes=159023104 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159251168 ingress_bytes=234520576 link_bytes=234520576 token_bytes=318274272 weight_movement_bytes=75497472 total_nvlink_bytes=393771744 weight_to_token_ratio=0.237209 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784430610e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44346 payload_rows_remote=38974 dup_rows=21190 dup_rows_remote=18620 dedup_ratio=0.323334 router_tokens_per_expert=[498, 539, 504, 492, 516, 479, 530, 526, 508, 500, 512, 499, 512, 486, 506, 494, 546, 511, 518, 542, 516, 525, 504, 480, 510, 513, 515, 502, 507, 519, 519, 500, 513, 526, 494, 461, 482, 514, 525, 456, 531, 521, 512, 558, 564, 477, 524, 532, 498, 527, 517, 534, 533, 530, 462, 498, 481, 511, 536, 523, 532, 547, 505, 516] payload_rows_per_dst=[5500, 5550, 5568, 5508, 5372, 5676, 5524, 5648] dup_rows_per_dst=[2668, 2484, 2716, 2662, 2570, 2762, 2674, 2654] combine_rows_per_src=[5500, 5550, 5568, 5508, 5372, 5676, 5524, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159612928 dispatch_meta_bytes=229504 combine_payload_bytes=159612928 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159842432 ingress_bytes=235110400 link_bytes=235110400 token_bytes=319455360 weight_movement_bytes=75497472 total_nvlink_bytes=394952832 weight_to_token_ratio=0.236332 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788918489e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159637504 dispatch_meta_bytes=230376 combine_payload_bytes=159637504 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159867880 ingress_bytes=184803328 link_bytes=184803328 token_bytes=319505384 weight_movement_bytes=25165824 total_nvlink_bytes=344671208 weight_to_token_ratio=0.078765 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406139798e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38938 dup_rows=21088 dup_rows_remote=18398 dedup_ratio=0.321777 router_tokens_per_expert=[511, 578, 501, 517, 516, 491, 510, 527, 477, 513, 540, 504, 547, 475, 549, 524, 519, 527, 477, 526, 494, 512, 480, 508, 516, 488, 522, 549, 494, 521, 513, 497, 485, 490, 527, 505, 538, 476, 526, 501, 500, 525, 489, 537, 516, 509, 504, 493, 517, 472, 539, 509, 528, 538, 516, 536, 510, 510, 519, 512, 472, 491, 510, 545] payload_rows_per_dst=[5628, 5616, 5524, 5510, 5548, 5558, 5600, 5464] dup_rows_per_dst=[2674, 2642, 2562, 2690, 2548, 2588, 2710, 2674] combine_rows_per_src=[5628, 5616, 5524, 5510, 5548, 5558, 5600, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44612 payload_rows_remote=39088 dup_rows=20924 dup_rows_remote=18264 dedup_ratio=0.319275 router_tokens_per_expert=[485, 492, 555, 509, 518, 505, 465, 520, 519, 557, 537, 492, 493, 510, 481, 550, 533, 527, 523, 503, 510, 522, 519, 496, 498, 483, 537, 534, 497, 514, 501, 534, 509, 477, 507, 516, 509, 511, 522, 519, 546, 475, 524, 512, 495, 519, 520, 501, 480, 517, 516, 489, 508, 563, 498, 493, 480, 547, 481, 517, 517, 522, 542, 517] payload_rows_per_dst=[5544, 5594, 5624, 5582, 5558, 5524, 5582, 5604] dup_rows_per_dst=[2554, 2684, 2642, 2614, 2582, 2660, 2546, 2642] combine_rows_per_src=[5544, 5594, 5624, 5582, 5558, 5524, 5582, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39042 dup_rows=20934 dup_rows_remote=18432 dedup_ratio=0.319427 router_tokens_per_expert=[511, 530, 507, 495, 497, 500, 522, 477, 500, 515, 496, 501, 506, 493, 511, 509, 533, 525, 506, 499, 553, 502, 480, 490, 509, 538, 548, 543, 517, 503, 534, 514, 536, 506, 487, 533, 548, 499, 542, 525, 496, 501, 508, 532, 489, 544, 538, 508, 529, 508, 511, 508, 504, 491, 496, 495, 496, 496, 504, 534, 513, 508, 518, 501] payload_rows_per_dst=[5562, 5560, 5586, 5692, 5600, 5568, 5562, 5472] dup_rows_per_dst=[2516, 2502, 2590, 2720, 2752, 2664, 2522, 2668] combine_rows_per_src=[5562, 5560, 5586, 5692, 5600, 5568, 5562, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44634 payload_rows_remote=38976 dup_rows=20902 dup_rows_remote=18220 dedup_ratio=0.318939 router_tokens_per_expert=[548, 511, 536, 497, 509, 521, 542, 490, 495, 546, 510, 530, 502, 545, 495, 516, 498, 494, 502, 500, 513, 489, 521, 498, 533, 524, 508, 506, 486, 521, 500, 529, 508, 497, 527, 555, 517, 477, 470, 500, 493, 506, 513, 521, 546, 501, 482, 538, 529, 487, 535, 507, 524, 515, 522, 475, 546, 516, 478, 528, 504, 521, 496, 519] payload_rows_per_dst=[5658, 5602, 5706, 5398, 5608, 5496, 5592, 5574] dup_rows_per_dst=[2682, 2676, 2796, 2420, 2658, 2554, 2596, 2520] combine_rows_per_src=[5658, 5602, 5706, 5398, 5608, 5496, 5592, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38880 dup_rows=21166 dup_rows_remote=18590 dedup_ratio=0.322968 router_tokens_per_expert=[509, 552, 541, 534, 525, 545, 492, 517, 511, 499, 516, 525, 512, 480, 537, 515, 496, 492, 504, 527, 492, 495, 529, 498, 508, 520, 513, 499, 507, 520, 545, 514, 516, 475, 524, 523, 464, 515, 531, 512, 477, 505, 502, 513, 543, 537, 525, 519, 508, 500, 512, 473, 501, 471, 522, 509, 459, 493, 532, 539, 511, 513, 573, 502] payload_rows_per_dst=[5596, 5598, 5490, 5558, 5516, 5536, 5464, 5612] dup_rows_per_dst=[2834, 2592, 2576, 2694, 2604, 2706, 2528, 2632] combine_rows_per_src=[5596, 5598, 5490, 5558, 5516, 5536, 5464, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=39010 dup_rows=21058 dup_rows_remote=18526 dedup_ratio=0.321320 router_tokens_per_expert=[512, 515, 526, 477, 521, 521, 492, 513, 550, 494, 528, 506, 553, 517, 474, 473, 532, 497, 537, 496, 493, 515, 493, 481, 491, 491, 548, 488, 540, 531, 521, 514, 511, 519, 477, 503, 531, 478, 575, 558, 522, 499, 514, 495, 546, 525, 538, 474, 496, 539, 487, 493, 494, 499, 518, 474, 539, 498, 525, 534, 533, 499, 517, 518] payload_rows_per_dst=[5594, 5534, 5482, 5608, 5612, 5564, 5468, 5616] dup_rows_per_dst=[2560, 2656, 2606, 2640, 2692, 2662, 2532, 2710] combine_rows_per_src=[5594, 5534, 5482, 5608, 5612, 5564, 5468, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159490048 dispatch_meta_bytes=229344 combine_payload_bytes=159490048 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159719392 ingress_bytes=209821696 link_bytes=209821696 token_bytes=319209440 weight_movement_bytes=50331648 total_nvlink_bytes=369541088 weight_to_token_ratio=0.157676 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596500671e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160104448 dispatch_meta_bytes=229408 combine_payload_bytes=160104448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160333856 ingress_bytes=185270272 link_bytes=185270272 token_bytes=320438304 weight_movement_bytes=25165824 total_nvlink_bytes=345604128 weight_to_token_ratio=0.078536 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409692702e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159916032 dispatch_meta_bytes=229896 combine_payload_bytes=159916032 prefetch_bytes=0 reduce_bytes=0 egress_bytes=160145928 ingress_bytes=159916032 link_bytes=160145928 token_bytes=320061960 weight_movement_bytes=0 total_nvlink_bytes=320061960 weight_to_token_ratio=0.000000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.218525474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44580 payload_rows_remote=38946 dup_rows=20956 dup_rows_remote=18134 dedup_ratio=0.319763 router_tokens_per_expert=[531, 461, 498, 498, 518, 496, 507, 534, 477, 494, 540, 548, 512, 516, 485, 516, 524, 510, 537, 504, 513, 516, 474, 518, 495, 530, 496, 524, 518, 509, 520, 547, 520, 544, 514, 519, 506, 509, 526, 499, 511, 509, 489, 509, 524, 508, 486, 507, 538, 495, 537, 527, 501, 505, 484, 507, 512, 510, 535, 510, 514, 537, 502, 508] payload_rows_per_dst=[5564, 5520, 5506, 5618, 5612, 5442, 5684, 5634] dup_rows_per_dst=[2522, 2458, 2486, 2756, 2662, 2440, 2810, 2822] combine_rows_per_src=[5564, 5520, 5506, 5618, 5612, 5442, 5684, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=228784 combine_payload_bytes=159645696 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159874480 ingress_bytes=209977344 link_bytes=209977344 token_bytes=319520176 weight_movement_bytes=50331648 total_nvlink_bytes=369851824 weight_to_token_ratio=0.157523 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597684972e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159252480 dispatch_meta_bytes=229880 combine_payload_bytes=159252480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159482360 ingress_bytes=184418304 link_bytes=184418304 token_bytes=318734840 weight_movement_bytes=25165824 total_nvlink_bytes=343900664 weight_to_token_ratio=0.078955 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403210210e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=230144 combine_payload_bytes=159784960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160015104 ingress_bytes=184950784 link_bytes=184950784 token_bytes=319800064 weight_movement_bytes=25165824 total_nvlink_bytes=344965888 weight_to_token_ratio=0.078692 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407261767e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44442 payload_rows_remote=38968 dup_rows=21094 dup_rows_remote=18488 dedup_ratio=0.321869 router_tokens_per_expert=[525, 546, 510, 504, 487, 498, 494, 504, 500, 535, 537, 502, 526, 509, 523, 477, 525, 500, 504, 509, 502, 527, 495, 533, 523, 485, 483, 539, 522, 556, 488, 492, 513, 529, 490, 497, 524, 521, 457, 509, 527, 475, 513, 503, 488, 496, 556, 521, 534, 504, 530, 507, 490, 545, 538, 520, 488, 503, 558, 512, 531, 494, 513, 522] payload_rows_per_dst=[5526, 5530, 5550, 5604, 5474, 5526, 5648, 5584] dup_rows_per_dst=[2610, 2688, 2640, 2572, 2606, 2632, 2688, 2658] combine_rows_per_src=[5526, 5530, 5550, 5604, 5474, 5526, 5648, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=228320 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159751136 ingress_bytes=184688640 link_bytes=184688640 token_bytes=319273952 weight_movement_bytes=25165824 total_nvlink_bytes=344439776 weight_to_token_ratio=0.078822 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405267154e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38802 dup_rows=21114 dup_rows_remote=18474 dedup_ratio=0.322174 router_tokens_per_expert=[518, 490, 482, 520, 542, 511, 469, 491, 505, 538, 510, 512, 499, 555, 557, 527, 492, 510, 534, 546, 495, 524, 497, 525, 482, 538, 471, 549, 510, 535, 507, 538, 519, 518, 529, 518, 504, 478, 529, 498, 510, 522, 525, 507, 506, 476, 528, 496, 491, 500, 481, 539, 512, 514, 463, 524, 546, 494, 530, 527, 538, 521, 477, 469] payload_rows_per_dst=[5494, 5640, 5528, 5620, 5534, 5516, 5536, 5554] dup_rows_per_dst=[2552, 2766, 2718, 2640, 2652, 2624, 2512, 2650] combine_rows_per_src=[5494, 5640, 5528, 5620, 5534, 5516, 5536, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38906 dup_rows=21000 dup_rows_remote=18344 dedup_ratio=0.320435 router_tokens_per_expert=[520, 494, 495, 514, 501, 505, 522, 537, 498, 495, 506, 528, 525, 501, 485, 512, 508, 528, 503, 507, 564, 520, 523, 498, 485, 504, 495, 533, 501, 493, 524, 527, 525, 527, 500, 517, 503, 519, 521, 514, 512, 527, 550, 545, 492, 506, 489, 522, 488, 517, 504, 528, 496, 490, 541, 488, 524, 520, 480, 541, 524, 520, 480, 507] payload_rows_per_dst=[5518, 5562, 5606, 5514, 5598, 5630, 5528, 5580] dup_rows_per_dst=[2658, 2538, 2696, 2610, 2654, 2656, 2576, 2612] combine_rows_per_src=[5518, 5562, 5606, 5514, 5598, 5630, 5528, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159612928 dispatch_meta_bytes=229824 combine_payload_bytes=159612928 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159842752 ingress_bytes=184778752 link_bytes=184778752 token_bytes=319455680 weight_movement_bytes=25165824 total_nvlink_bytes=344621504 weight_to_token_ratio=0.078777 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405952803e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44384 payload_rows_remote=38810 dup_rows=21152 dup_rows_remote=18516 dedup_ratio=0.322754 router_tokens_per_expert=[523, 526, 517, 493, 551, 525, 505, 523, 509, 508, 508, 505, 536, 521, 482, 536, 525, 513, 525, 499, 543, 520, 548, 472, 471, 527, 496, 501, 524, 512, 506, 546, 513, 530, 492, 438, 538, 523, 526, 545, 523, 523, 466, 525, 499, 499, 539, 493, 504, 512, 478, 510, 499, 511, 457, 530, 510, 499, 498, 515, 504, 541, 527, 505] payload_rows_per_dst=[5576, 5574, 5606, 5570, 5514, 5498, 5484, 5562] dup_rows_per_dst=[2750, 2636, 2684, 2596, 2696, 2636, 2518, 2636] combine_rows_per_src=[5576, 5574, 5606, 5570, 5514, 5498, 5484, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38918 dup_rows=21026 dup_rows_remote=18264 dedup_ratio=0.320831 router_tokens_per_expert=[544, 477, 537, 485, 526, 521, 483, 524, 508, 554, 494, 517, 514, 491, 488, 513, 515, 530, 511, 491, 493, 490, 532, 509, 488, 479, 502, 505, 513, 526, 471, 547, 513, 529, 468, 493, 497, 523, 557, 515, 505, 514, 510, 536, 526, 524, 538, 515, 519, 532, 512, 537, 499, 495, 500, 497, 542, 521, 509, 536, 543, 498, 489, 498] payload_rows_per_dst=[5592, 5494, 5584, 5524, 5428, 5688, 5588, 5612] dup_rows_per_dst=[2762, 2680, 2566, 2538, 2578, 2648, 2594, 2660] combine_rows_per_src=[5592, 5494, 5584, 5524, 5428, 5688, 5588, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44496 payload_rows_remote=38836 dup_rows=21040 dup_rows_remote=18434 dedup_ratio=0.321045 router_tokens_per_expert=[515, 570, 520, 531, 522, 503, 514, 488, 526, 520, 493, 496, 492, 542, 474, 500, 493, 498, 550, 530, 502, 530, 500, 530, 507, 517, 503, 483, 515, 512, 520, 498, 500, 528, 514, 499, 541, 508, 489, 494, 531, 487, 525, 518, 520, 529, 521, 499, 560, 491, 509, 513, 521, 496, 547, 490, 516, 478, 496, 511, 519, 500, 522, 502] payload_rows_per_dst=[5646, 5516, 5660, 5552, 5558, 5560, 5534, 5470] dup_rows_per_dst=[2680, 2570, 2606, 2558, 2588, 2700, 2720, 2618] combine_rows_per_src=[5646, 5516, 5660, 5552, 5558, 5560, 5534, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229000 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159587976 ingress_bytes=209690624 link_bytes=209690624 token_bytes=318946952 weight_movement_bytes=50331648 total_nvlink_bytes=369278600 weight_to_token_ratio=0.157806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595503365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158932992 dispatch_meta_bytes=229104 combine_payload_bytes=158932992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159162096 ingress_bytes=184098816 link_bytes=184098816 token_bytes=318095088 weight_movement_bytes=25165824 total_nvlink_bytes=343260912 weight_to_token_ratio=0.079114 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400779275e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44600 payload_rows_remote=39142 dup_rows=20936 dup_rows_remote=18318 dedup_ratio=0.319458 router_tokens_per_expert=[489, 528, 464, 486, 523, 521, 521, 484, 506, 508, 548, 469, 537, 526, 525, 535, 497, 500, 489, 534, 541, 492, 507, 511, 505, 506, 503, 511, 515, 531, 538, 495, 544, 510, 530, 503, 511, 529, 504, 540, 553, 549, 497, 491, 532, 515, 557, 512, 476, 528, 473, 527, 522, 512, 496, 504, 472, 523, 482, 498, 481, 502, 533, 517] payload_rows_per_dst=[5500, 5570, 5572, 5616, 5634, 5724, 5458, 5526] dup_rows_per_dst=[2532, 2738, 2570, 2592, 2708, 2688, 2618, 2490] combine_rows_per_src=[5500, 5570, 5572, 5616, 5634, 5724, 5458, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158965760 dispatch_meta_bytes=229304 combine_payload_bytes=158965760 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159195064 ingress_bytes=234463232 link_bytes=234463232 token_bytes=318160824 weight_movement_bytes=75497472 total_nvlink_bytes=393658296 weight_to_token_ratio=0.237293 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783994288e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44730 payload_rows_remote=39114 dup_rows=20806 dup_rows_remote=18124 dedup_ratio=0.317474 router_tokens_per_expert=[509, 512, 508, 502, 530, 542, 527, 504, 503, 523, 484, 488, 512, 497, 512, 488, 495, 490, 508, 509, 564, 508, 484, 487, 534, 523, 520, 505, 517, 526, 532, 462, 511, 521, 497, 523, 479, 520, 520, 515, 477, 517, 549, 520, 528, 488, 533, 529, 527, 477, 512, 519, 527, 519, 508, 502, 516, 537, 517, 530, 489, 510, 511, 535] payload_rows_per_dst=[5614, 5456, 5552, 5654, 5662, 5630, 5546, 5616] dup_rows_per_dst=[2654, 2502, 2538, 2584, 2510, 2652, 2684, 2682] combine_rows_per_src=[5614, 5456, 5552, 5654, 5662, 5630, 5546, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=228728 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159636856 ingress_bytes=184573952 link_bytes=184573952 token_bytes=319044984 weight_movement_bytes=25165824 total_nvlink_bytes=344210808 weight_to_token_ratio=0.078879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404394511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159072256 dispatch_meta_bytes=229080 combine_payload_bytes=159072256 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159301336 ingress_bytes=209403904 link_bytes=209403904 token_bytes=318373592 weight_movement_bytes=50331648 total_nvlink_bytes=368705240 weight_to_token_ratio=0.158090 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593321757e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160325632 dispatch_meta_bytes=229840 combine_payload_bytes=160325632 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160555472 ingress_bytes=185491456 link_bytes=185491456 token_bytes=320881104 weight_movement_bytes=25165824 total_nvlink_bytes=346046928 weight_to_token_ratio=0.078427 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411375657e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38964 dup_rows=21036 dup_rows_remote=18400 dedup_ratio=0.320984 router_tokens_per_expert=[510, 527, 526, 501, 504, 493, 505, 550, 542, 519, 505, 475, 512, 506, 520, 539, 555, 509, 514, 531, 496, 508, 512, 510, 540, 514, 513, 466, 491, 505, 509, 503, 515, 505, 520, 454, 486, 547, 553, 506, 524, 489, 523, 499, 498, 498, 548, 510, 527, 547, 520, 510, 529, 493, 483, 503, 506, 498, 504, 488, 539, 525, 492, 519] payload_rows_per_dst=[5554, 5634, 5558, 5588, 5536, 5524, 5604, 5502] dup_rows_per_dst=[2678, 2602, 2712, 2494, 2636, 2654, 2620, 2640] combine_rows_per_src=[5554, 5634, 5558, 5588, 5536, 5524, 5604, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160210944 dispatch_meta_bytes=228952 combine_payload_bytes=160210944 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=160439896 ingress_bytes=260874240 link_bytes=260874240 token_bytes=320650840 weight_movement_bytes=100663296 total_nvlink_bytes=421314136 weight_to_token_ratio=0.313934 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.984951543e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44334 payload_rows_remote=38738 dup_rows=21202 dup_rows_remote=18542 dedup_ratio=0.323517 router_tokens_per_expert=[502, 516, 530, 518, 488, 505, 494, 502, 508, 519, 492, 544, 519, 495, 518, 503, 504, 499, 514, 470, 518, 521, 547, 530, 531, 525, 514, 509, 492, 512, 548, 497, 508, 562, 529, 494, 531, 510, 504, 485, 525, 507, 502, 533, 551, 508, 534, 503, 532, 489, 499, 477, 475, 517, 520, 479, 505, 506, 470, 527, 552, 524, 515, 511] payload_rows_per_dst=[5516, 5540, 5536, 5596, 5594, 5578, 5502, 5472] dup_rows_per_dst=[2594, 2656, 2670, 2660, 2652, 2748, 2474, 2748] combine_rows_per_src=[5516, 5540, 5536, 5596, 5594, 5578, 5502, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38934 dup_rows=21082 dup_rows_remote=18408 dedup_ratio=0.321686 router_tokens_per_expert=[509, 503, 521, 494, 512, 507, 518, 503, 502, 499, 517, 494, 528, 519, 505, 505, 497, 544, 509, 483, 499, 504, 524, 530, 529, 534, 521, 509, 487, 499, 548, 510, 526, 493, 528, 511, 516, 479, 543, 504, 545, 559, 526, 463, 497, 523, 519, 465, 508, 500, 547, 531, 537, 532, 494, 479, 508, 554, 469, 516, 491, 473, 544, 525] payload_rows_per_dst=[5598, 5528, 5586, 5562, 5496, 5520, 5622, 5542] dup_rows_per_dst=[2536, 2610, 2594, 2712, 2704, 2674, 2634, 2618] combine_rows_per_src=[5598, 5528, 5586, 5562, 5496, 5520, 5622, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229456 combine_payload_bytes=159596544 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159826000 ingress_bytes=235094016 link_bytes=235094016 token_bytes=319422544 weight_movement_bytes=75497472 total_nvlink_bytes=394920016 weight_to_token_ratio=0.236356 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788793826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44632 payload_rows_remote=39046 dup_rows=20904 dup_rows_remote=18186 dedup_ratio=0.318970 router_tokens_per_expert=[497, 534, 526, 536, 513, 521, 474, 510, 494, 493, 525, 510, 525, 519, 531, 555, 485, 514, 515, 540, 502, 515, 524, 489, 540, 510, 495, 534, 525, 514, 516, 476, 491, 487, 487, 521, 531, 522, 532, 534, 503, 567, 467, 483, 485, 499, 479, 494, 543, 505, 493, 510, 523, 545, 516, 538, 506, 546, 506, 519, 489, 498, 512, 480] payload_rows_per_dst=[5618, 5586, 5632, 5622, 5588, 5394, 5638, 5554] dup_rows_per_dst=[2604, 2718, 2536, 2598, 2622, 2560, 2708, 2558] combine_rows_per_src=[5618, 5586, 5632, 5622, 5588, 5394, 5638, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44154 payload_rows_remote=38686 dup_rows=21382 dup_rows_remote=18672 dedup_ratio=0.326263 router_tokens_per_expert=[484, 514, 505, 527, 498, 509, 486, 515, 560, 518, 532, 531, 515, 509, 499, 491, 506, 508, 544, 539, 512, 508, 502, 490, 500, 493, 530, 530, 523, 478, 470, 544, 530, 499, 475, 510, 501, 508, 528, 509, 509, 483, 535, 542, 541, 524, 498, 460, 501, 532, 511, 506, 536, 555, 510, 492, 553, 466, 520, 527, 509, 506, 514, 508] payload_rows_per_dst=[5468, 5566, 5652, 5414, 5556, 5594, 5588, 5316] dup_rows_per_dst=[2710, 2744, 2964, 2264, 2742, 2786, 2698, 2474] combine_rows_per_src=[5468, 5566, 5652, 5414, 5556, 5594, 5588, 5316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38824 dup_rows=21116 dup_rows_remote=18414 dedup_ratio=0.322205 router_tokens_per_expert=[486, 499, 534, 496, 505, 516, 529, 512, 531, 497, 511, 519, 535, 557, 484, 533, 530, 514, 499, 540, 504, 550, 509, 503, 514, 502, 553, 518, 496, 530, 509, 516, 475, 519, 531, 500, 548, 531, 511, 505, 507, 489, 509, 499, 489, 477, 530, 497, 536, 508, 503, 509, 498, 496, 530, 500, 496, 522, 504, 512, 493, 532, 508, 473] payload_rows_per_dst=[5532, 5598, 5596, 5558, 5558, 5482, 5598, 5498] dup_rows_per_dst=[2622, 2736, 2702, 2718, 2682, 2512, 2562, 2582] combine_rows_per_src=[5532, 5598, 5596, 5558, 5558, 5482, 5598, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158670848 dispatch_meta_bytes=229120 combine_payload_bytes=158670848 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158899968 ingress_bytes=234168320 link_bytes=234168320 token_bytes=317570816 weight_movement_bytes=75497472 total_nvlink_bytes=393068288 weight_to_token_ratio=0.237734 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781750349e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159473664 dispatch_meta_bytes=229368 combine_payload_bytes=159473664 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159703032 ingress_bytes=209805312 link_bytes=209805312 token_bytes=319176696 weight_movement_bytes=50331648 total_nvlink_bytes=369508344 weight_to_token_ratio=0.157692 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596376008e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38916 dup_rows=21116 dup_rows_remote=18480 dedup_ratio=0.322205 router_tokens_per_expert=[533, 497, 527, 515, 539, 484, 506, 519, 505, 562, 471, 486, 543, 490, 465, 497, 529, 485, 494, 497, 530, 546, 518, 460, 521, 498, 548, 527, 476, 560, 505, 546, 501, 526, 517, 512, 505, 500, 515, 514, 510, 504, 512, 520, 516, 503, 523, 522, 512, 513, 534, 494, 498, 503, 497, 519, 492, 502, 519, 486, 511, 548, 513, 548] payload_rows_per_dst=[5548, 5524, 5558, 5602, 5496, 5592, 5504, 5596] dup_rows_per_dst=[2692, 2514, 2560, 2760, 2684, 2628, 2636, 2642] combine_rows_per_src=[5548, 5524, 5558, 5602, 5496, 5592, 5504, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38742 dup_rows=21100 dup_rows_remote=18234 dedup_ratio=0.321960 router_tokens_per_expert=[472, 505, 518, 549, 542, 512, 502, 509, 495, 552, 540, 526, 510, 510, 535, 521, 477, 480, 506, 504, 515, 509, 557, 523, 493, 507, 486, 521, 513, 502, 527, 508, 513, 458, 509, 527, 494, 558, 504, 510, 509, 531, 531, 523, 472, 522, 532, 519, 486, 512, 534, 525, 500, 510, 499, 507, 535, 503, 490, 528, 521, 495, 490, 495] payload_rows_per_dst=[5628, 5420, 5512, 5542, 5466, 5644, 5530, 5694] dup_rows_per_dst=[2590, 2512, 2630, 2512, 2548, 2820, 2622, 2866] combine_rows_per_src=[5628, 5420, 5512, 5542, 5466, 5644, 5530, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159932416 dispatch_meta_bytes=228928 combine_payload_bytes=159932416 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160161344 ingress_bytes=210264064 link_bytes=210264064 token_bytes=320093760 weight_movement_bytes=50331648 total_nvlink_bytes=370425408 weight_to_token_ratio=0.157240 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599866580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158457856 dispatch_meta_bytes=229432 combine_payload_bytes=158457856 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158687288 ingress_bytes=183623680 link_bytes=183623680 token_bytes=317145144 weight_movement_bytes=25165824 total_nvlink_bytes=342310968 weight_to_token_ratio=0.079351 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397164039e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159023104 dispatch_meta_bytes=228952 combine_payload_bytes=159023104 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159252056 ingress_bytes=184188928 link_bytes=184188928 token_bytes=318275160 weight_movement_bytes=25165824 total_nvlink_bytes=343440984 weight_to_token_ratio=0.079069 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401464924e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159399936 dispatch_meta_bytes=229584 combine_payload_bytes=159399936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159629520 ingress_bytes=184565760 link_bytes=184565760 token_bytes=319029456 weight_movement_bytes=25165824 total_nvlink_bytes=344195280 weight_to_token_ratio=0.078882 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404332180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44526 payload_rows_remote=38912 dup_rows=21010 dup_rows_remote=18420 dedup_ratio=0.320587 router_tokens_per_expert=[488, 553, 507, 476, 510, 488, 504, 487, 553, 528, 520, 530, 495, 480, 492, 515, 493, 499, 526, 469, 507, 537, 515, 507, 522, 528, 535, 531, 517, 507, 529, 524, 545, 519, 542, 482, 502, 504, 503, 505, 509, 546, 472, 516, 477, 492, 520, 515, 491, 533, 520, 540, 509, 491, 501, 491, 527, 550, 539, 517, 495, 495, 535, 513] payload_rows_per_dst=[5522, 5652, 5466, 5602, 5614, 5522, 5552, 5596] dup_rows_per_dst=[2504, 2574, 2640, 2784, 2590, 2572, 2600, 2746] combine_rows_per_src=[5522, 5652, 5466, 5602, 5614, 5522, 5552, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158687232 dispatch_meta_bytes=227904 combine_payload_bytes=158687232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158915136 ingress_bytes=183853056 link_bytes=183853056 token_bytes=317602368 weight_movement_bytes=25165824 total_nvlink_bytes=342768192 weight_to_token_ratio=0.079237 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398909326e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44540 payload_rows_remote=38932 dup_rows=20996 dup_rows_remote=18286 dedup_ratio=0.320374 router_tokens_per_expert=[486, 521, 504, 498, 535, 471, 483, 523, 469, 553, 485, 525, 568, 521, 499, 523, 548, 489, 493, 518, 509, 485, 484, 475, 474, 548, 509, 510, 549, 539, 507, 523, 536, 516, 507, 528, 523, 535, 537, 504, 500, 516, 494, 496, 498, 524, 525, 512, 500, 488, 526, 546, 546, 509, 536, 517, 511, 496, 506, 496, 536, 489, 505, 486] payload_rows_per_dst=[5512, 5602, 5510, 5608, 5666, 5542, 5626, 5474] dup_rows_per_dst=[2530, 2684, 2492, 2710, 2706, 2588, 2710, 2576] combine_rows_per_src=[5512, 5602, 5510, 5608, 5666, 5542, 5626, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38790 dup_rows=21064 dup_rows_remote=18460 dedup_ratio=0.321411 router_tokens_per_expert=[517, 541, 507, 528, 547, 505, 504, 520, 478, 500, 550, 544, 498, 523, 501, 491, 520, 477, 471, 474, 496, 474, 518, 537, 513, 512, 524, 485, 530, 513, 480, 503, 510, 459, 450, 522, 499, 514, 523, 516, 514, 558, 518, 501, 522, 483, 549, 498, 542, 504, 504, 546, 480, 542, 506, 499, 548, 553, 498, 537, 553, 495, 511, 533] payload_rows_per_dst=[5598, 5502, 5478, 5506, 5454, 5682, 5562, 5690] dup_rows_per_dst=[2740, 2668, 2456, 2614, 2532, 2604, 2684, 2766] combine_rows_per_src=[5598, 5502, 5478, 5506, 5454, 5682, 5562, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159383552 dispatch_meta_bytes=229328 combine_payload_bytes=159383552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159612880 ingress_bytes=184549376 link_bytes=184549376 token_bytes=318996432 weight_movement_bytes=25165824 total_nvlink_bytes=344162256 weight_to_token_ratio=0.078891 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404207516e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44512 payload_rows_remote=38988 dup_rows=21024 dup_rows_remote=18404 dedup_ratio=0.320801 router_tokens_per_expert=[526, 495, 539, 541, 510, 513, 537, 478, 475, 530, 465, 510, 511, 542, 525, 514, 500, 495, 503, 515, 493, 519, 504, 517, 540, 524, 466, 511, 524, 505, 533, 506, 536, 526, 498, 519, 510, 519, 542, 509, 509, 497, 476, 520, 512, 546, 495, 480, 475, 484, 552, 500, 489, 511, 513, 517, 489, 531, 487, 544, 499, 575, 523, 519] payload_rows_per_dst=[5546, 5524, 5532, 5574, 5702, 5530, 5492, 5612] dup_rows_per_dst=[2732, 2620, 2560, 2644, 2616, 2540, 2590, 2722] combine_rows_per_src=[5546, 5524, 5532, 5574, 5702, 5530, 5492, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44716 payload_rows_remote=38928 dup_rows=20820 dup_rows_remote=18018 dedup_ratio=0.317688 router_tokens_per_expert=[515, 523, 555, 548, 545, 535, 490, 491, 508, 462, 499, 524, 528, 509, 558, 529, 506, 559, 508, 498, 487, 522, 477, 521, 506, 482, 520, 541, 495, 501, 526, 508, 501, 521, 497, 496, 545, 496, 500, 513, 572, 514, 544, 529, 501, 498, 503, 512, 486, 542, 513, 512, 531, 489, 523, 471, 505, 502, 507, 524, 472, 479, 504, 490] payload_rows_per_dst=[5788, 5576, 5710, 5568, 5404, 5638, 5522, 5510] dup_rows_per_dst=[2802, 2520, 2750, 2542, 2430, 2708, 2612, 2456] combine_rows_per_src=[5788, 5576, 5710, 5568, 5404, 5638, 5522, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38976 dup_rows=21000 dup_rows_remote=18368 dedup_ratio=0.320435 router_tokens_per_expert=[492, 495, 524, 435, 506, 507, 482, 517, 515, 505, 494, 486, 514, 506, 539, 533, 513, 522, 515, 523, 504, 501, 485, 533, 526, 541, 523, 529, 529, 487, 530, 450, 541, 505, 525, 509, 532, 480, 524, 502, 493, 493, 539, 521, 481, 520, 516, 502, 528, 539, 537, 512, 535, 497, 502, 550, 537, 486, 497, 508, 523, 523, 517, 533] payload_rows_per_dst=[5478, 5568, 5560, 5592, 5602, 5438, 5726, 5572] dup_rows_per_dst=[2438, 2616, 2632, 2638, 2634, 2692, 2674, 2676] combine_rows_per_src=[5478, 5568, 5560, 5592, 5602, 5438, 5726, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159465472 dispatch_meta_bytes=228872 combine_payload_bytes=159465472 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159694344 ingress_bytes=234962944 link_bytes=234962944 token_bytes=319159816 weight_movement_bytes=75497472 total_nvlink_bytes=394657288 weight_to_token_ratio=0.236551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787796519e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44402 payload_rows_remote=38850 dup_rows=21134 dup_rows_remote=18598 dedup_ratio=0.322479 router_tokens_per_expert=[533, 553, 504, 513, 491, 500, 504, 521, 532, 541, 497, 544, 534, 504, 511, 498, 493, 508, 531, 524, 503, 548, 520, 495, 509, 479, 569, 493, 504, 539, 492, 487, 510, 493, 519, 534, 548, 486, 539, 557, 488, 506, 495, 553, 467, 505, 490, 518, 495, 545, 502, 504, 532, 478, 537, 451, 533, 490, 519, 468, 484, 536, 498, 514] payload_rows_per_dst=[5568, 5570, 5566, 5544, 5654, 5496, 5552, 5452] dup_rows_per_dst=[2670, 2752, 2678, 2600, 2718, 2548, 2536, 2632] combine_rows_per_src=[5568, 5570, 5566, 5544, 5654, 5496, 5552, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158883840 dispatch_meta_bytes=229000 combine_payload_bytes=158883840 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159112840 ingress_bytes=184049664 link_bytes=184049664 token_bytes=317996680 weight_movement_bytes=25165824 total_nvlink_bytes=343162504 weight_to_token_ratio=0.079139 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400405285e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=229568 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159924416 ingress_bytes=235192320 link_bytes=235192320 token_bytes=319619264 weight_movement_bytes=75497472 total_nvlink_bytes=395116736 weight_to_token_ratio=0.236211 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789541805e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44238 payload_rows_remote=38732 dup_rows=21298 dup_rows_remote=18676 dedup_ratio=0.324982 router_tokens_per_expert=[550, 526, 502, 479, 508, 492, 517, 505, 491, 500, 498, 485, 529, 510, 498, 480, 512, 479, 540, 509, 495, 526, 522, 552, 511, 531, 541, 490, 504, 518, 527, 507, 515, 529, 513, 523, 517, 522, 527, 544, 525, 524, 504, 488, 534, 543, 518, 506, 498, 509, 527, 491, 492, 523, 511, 492, 546, 510, 481, 487, 547, 525, 483, 480] payload_rows_per_dst=[5270, 5304, 5278, 5720, 5638, 5766, 5756, 5506] dup_rows_per_dst=[2370, 2262, 2342, 3056, 2742, 2934, 2970, 2622] combine_rows_per_src=[5270, 5304, 5278, 5720, 5638, 5766, 5756, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159449088 dispatch_meta_bytes=227784 combine_payload_bytes=159449088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159676872 ingress_bytes=184614912 link_bytes=184614912 token_bytes=319125960 weight_movement_bytes=25165824 total_nvlink_bytes=344291784 weight_to_token_ratio=0.078859 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404706170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=229376 combine_payload_bytes=159645696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159875072 ingress_bytes=184811520 link_bytes=184811520 token_bytes=319520768 weight_movement_bytes=25165824 total_nvlink_bytes=344686592 weight_to_token_ratio=0.078761 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406202129e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159129600 dispatch_meta_bytes=229792 combine_payload_bytes=159129600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159359392 ingress_bytes=184295424 link_bytes=184295424 token_bytes=318488992 weight_movement_bytes=25165824 total_nvlink_bytes=343654816 weight_to_token_ratio=0.079016 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402275235e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44424 payload_rows_remote=38846 dup_rows=21112 dup_rows_remote=18556 dedup_ratio=0.322144 router_tokens_per_expert=[495, 521, 527, 504, 518, 483, 498, 537, 479, 539, 569, 520, 521, 547, 511, 496, 491, 485, 534, 534, 521, 523, 508, 535, 533, 511, 501, 495, 491, 509, 529, 510, 473, 511, 531, 498, 510, 495, 540, 509, 509, 486, 514, 481, 532, 512, 481, 531, 505, 522, 522, 522, 505, 491, 526, 507, 512, 529, 517, 480, 506, 521, 489, 526] payload_rows_per_dst=[5558, 5606, 5636, 5502, 5578, 5500, 5518, 5526] dup_rows_per_dst=[2608, 2758, 2626, 2656, 2556, 2592, 2682, 2634] combine_rows_per_src=[5558, 5606, 5636, 5502, 5578, 5500, 5518, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158646272 dispatch_meta_bytes=229632 combine_payload_bytes=158646272 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158875904 ingress_bytes=234143744 link_bytes=234143744 token_bytes=317522176 weight_movement_bytes=75497472 total_nvlink_bytes=393019648 weight_to_token_ratio=0.237771 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781563354e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44584 payload_rows_remote=38996 dup_rows=20952 dup_rows_remote=18452 dedup_ratio=0.319702 router_tokens_per_expert=[505, 493, 558, 500, 528, 521, 493, 518, 525, 483, 522, 545, 520, 515, 503, 526, 531, 499, 568, 502, 518, 508, 551, 541, 484, 507, 498, 497, 535, 479, 531, 513, 480, 533, 503, 497, 511, 530, 504, 506, 485, 485, 538, 514, 520, 534, 493, 484, 504, 546, 532, 497, 516, 531, 512, 499, 523, 488, 499, 469, 524, 492, 474, 528] payload_rows_per_dst=[5632, 5634, 5684, 5588, 5502, 5466, 5618, 5460] dup_rows_per_dst=[2600, 2644, 2752, 2500, 2626, 2640, 2656, 2534] combine_rows_per_src=[5632, 5634, 5684, 5588, 5502, 5466, 5618, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=229608 combine_payload_bytes=159113216 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159342824 ingress_bytes=234610688 link_bytes=234610688 token_bytes=318456040 weight_movement_bytes=75497472 total_nvlink_bytes=393953512 weight_to_token_ratio=0.237073 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785116258e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44442 payload_rows_remote=38830 dup_rows=21094 dup_rows_remote=18446 dedup_ratio=0.321869 router_tokens_per_expert=[535, 490, 511, 490, 535, 523, 489, 471, 505, 552, 505, 517, 517, 480, 517, 511, 480, 523, 507, 525, 493, 534, 516, 493, 513, 483, 533, 539, 551, 533, 538, 462, 520, 526, 495, 467, 536, 519, 534, 531, 532, 546, 509, 540, 506, 483, 488, 526, 492, 516, 508, 483, 560, 528, 520, 465, 520, 514, 504, 508, 519, 525, 489, 488] payload_rows_per_dst=[5482, 5610, 5522, 5602, 5554, 5612, 5570, 5490] dup_rows_per_dst=[2606, 2598, 2620, 2702, 2702, 2648, 2574, 2644] combine_rows_per_src=[5482, 5610, 5522, 5602, 5554, 5612, 5570, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38858 dup_rows=21088 dup_rows_remote=18430 dedup_ratio=0.321777 router_tokens_per_expert=[531, 500, 542, 530, 547, 522, 513, 488, 506, 532, 523, 477, 549, 543, 497, 497, 519, 545, 532, 536, 494, 464, 525, 469, 529, 487, 488, 533, 510, 522, 525, 538, 462, 522, 520, 481, 519, 539, 506, 495, 510, 507, 505, 502, 528, 546, 528, 488, 471, 528, 537, 478, 529, 491, 508, 491, 493, 475, 541, 514, 541, 531, 497, 472] payload_rows_per_dst=[5588, 5590, 5546, 5542, 5588, 5544, 5534, 5516] dup_rows_per_dst=[2758, 2658, 2622, 2722, 2500, 2684, 2532, 2612] combine_rows_per_src=[5588, 5590, 5546, 5542, 5588, 5544, 5534, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44494 payload_rows_remote=38908 dup_rows=21042 dup_rows_remote=18296 dedup_ratio=0.321075 router_tokens_per_expert=[505, 536, 514, 523, 515, 514, 470, 526, 569, 527, 483, 512, 525, 532, 521, 519, 508, 509, 495, 523, 517, 488, 521, 560, 517, 507, 526, 502, 475, 490, 483, 538, 563, 494, 504, 522, 490, 525, 487, 495, 545, 487, 529, 480, 504, 504, 520, 507, 489, 482, 517, 515, 483, 536, 491, 507, 507, 515, 555, 537, 507, 523, 506, 492] payload_rows_per_dst=[5586, 5700, 5622, 5492, 5572, 5468, 5468, 5586] dup_rows_per_dst=[2746, 2802, 2494, 2584, 2634, 2684, 2572, 2526] combine_rows_per_src=[5586, 5700, 5622, 5492, 5572, 5468, 5468, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44270 payload_rows_remote=38750 dup_rows=21266 dup_rows_remote=18558 dedup_ratio=0.324493 router_tokens_per_expert=[544, 525, 486, 539, 506, 549, 496, 527, 503, 533, 519, 527, 515, 505, 511, 515, 537, 514, 493, 517, 522, 500, 556, 475, 486, 492, 526, 477, 509, 492, 521, 545, 521, 527, 523, 520, 495, 533, 516, 505, 523, 496, 470, 516, 519, 517, 503, 491, 493, 497, 516, 487, 476, 558, 528, 506, 501, 481, 519, 529, 491, 503, 526, 520] payload_rows_per_dst=[5584, 5566, 5520, 5480, 5562, 5504, 5504, 5550] dup_rows_per_dst=[2760, 2690, 2708, 2616, 2718, 2566, 2618, 2590] combine_rows_per_src=[5584, 5566, 5520, 5480, 5562, 5504, 5504, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159727616 dispatch_meta_bytes=229792 combine_payload_bytes=159727616 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159957408 ingress_bytes=184893440 link_bytes=184893440 token_bytes=319685024 weight_movement_bytes=25165824 total_nvlink_bytes=344850848 weight_to_token_ratio=0.078721 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406825446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44556 payload_rows_remote=39018 dup_rows=20980 dup_rows_remote=18386 dedup_ratio=0.320129 router_tokens_per_expert=[471, 518, 519, 510, 512, 548, 512, 482, 529, 502, 512, 525, 518, 471, 515, 491, 517, 505, 528, 521, 539, 499, 527, 563, 553, 499, 519, 509, 527, 484, 509, 468, 487, 503, 509, 487, 479, 516, 503, 541, 519, 517, 533, 515, 488, 537, 485, 504, 496, 517, 513, 513, 500, 521, 493, 513, 506, 520, 489, 526, 535, 533, 505, 563] payload_rows_per_dst=[5496, 5530, 5630, 5564, 5532, 5626, 5538, 5640] dup_rows_per_dst=[2648, 2596, 2768, 2572, 2518, 2570, 2594, 2714] combine_rows_per_src=[5496, 5530, 5630, 5564, 5532, 5626, 5538, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159047680 dispatch_meta_bytes=229104 combine_payload_bytes=159047680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159276784 ingress_bytes=184213504 link_bytes=184213504 token_bytes=318324464 weight_movement_bytes=25165824 total_nvlink_bytes=343490288 weight_to_token_ratio=0.079057 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401651918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38740 dup_rows=21146 dup_rows_remote=18422 dedup_ratio=0.322662 router_tokens_per_expert=[521, 503, 529, 480, 500, 467, 505, 505, 500, 529, 531, 515, 488, 530, 543, 499, 499, 513, 524, 506, 527, 517, 492, 530, 554, 545, 507, 498, 498, 549, 520, 523, 544, 454, 477, 506, 487, 504, 517, 481, 518, 508, 504, 521, 524, 530, 470, 519, 501, 495, 492, 527, 513, 491, 496, 555, 541, 520, 529, 491, 527, 561, 491, 527] payload_rows_per_dst=[5490, 5572, 5538, 5570, 5386, 5616, 5568, 5650] dup_rows_per_dst=[2530, 2698, 2678, 2676, 2554, 2624, 2662, 2724] combine_rows_per_src=[5490, 5572, 5538, 5570, 5386, 5616, 5568, 5650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159162368 dispatch_meta_bytes=229152 combine_payload_bytes=159162368 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159391520 ingress_bytes=209494016 link_bytes=209494016 token_bytes=318553888 weight_movement_bytes=50331648 total_nvlink_bytes=368885536 weight_to_token_ratio=0.158000 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594007405e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159367168 dispatch_meta_bytes=228816 combine_payload_bytes=159367168 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159595984 ingress_bytes=209698816 link_bytes=209698816 token_bytes=318963152 weight_movement_bytes=50331648 total_nvlink_bytes=369294800 weight_to_token_ratio=0.157798 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595565696e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44638 payload_rows_remote=38972 dup_rows=20898 dup_rows_remote=18212 dedup_ratio=0.318878 router_tokens_per_expert=[496, 505, 521, 525, 518, 505, 482, 552, 509, 539, 509, 543, 515, 508, 512, 523, 541, 527, 543, 516, 495, 496, 469, 480, 512, 485, 540, 513, 512, 501, 516, 487, 544, 527, 478, 508, 525, 528, 547, 519, 513, 519, 454, 530, 517, 501, 475, 505, 504, 501, 509, 510, 513, 512, 511, 478, 501, 525, 531, 497, 471, 550, 540, 530] payload_rows_per_dst=[5624, 5602, 5560, 5538, 5666, 5458, 5508, 5682] dup_rows_per_dst=[2584, 2714, 2574, 2594, 2686, 2570, 2568, 2608] combine_rows_per_src=[5624, 5602, 5560, 5538, 5666, 5458, 5508, 5682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158720000 dispatch_meta_bytes=229232 combine_payload_bytes=158720000 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=158949232 ingress_bytes=234217472 link_bytes=234217472 token_bytes=317669232 weight_movement_bytes=75497472 total_nvlink_bytes=393166704 weight_to_token_ratio=0.237661 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782124339e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=229616 combine_payload_bytes=159817728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160047344 ingress_bytes=184983552 link_bytes=184983552 token_bytes=319865072 weight_movement_bytes=25165824 total_nvlink_bytes=345030896 weight_to_token_ratio=0.078676 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407511094e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158679040 dispatch_meta_bytes=228648 combine_payload_bytes=158679040 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=158907688 ingress_bytes=234176512 link_bytes=234176512 token_bytes=317586728 weight_movement_bytes=75497472 total_nvlink_bytes=393084200 weight_to_token_ratio=0.237722 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781812680e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44426 payload_rows_remote=38828 dup_rows=21110 dup_rows_remote=18394 dedup_ratio=0.322113 router_tokens_per_expert=[497, 482, 533, 515, 500, 514, 525, 538, 576, 500, 511, 512, 481, 520, 462, 521, 480, 504, 549, 511, 500, 523, 519, 525, 534, 487, 539, 502, 525, 519, 526, 525, 527, 532, 542, 547, 461, 526, 539, 466, 488, 517, 545, 530, 478, 515, 476, 446, 490, 515, 536, 496, 536, 529, 489, 500, 521, 501, 537, 515, 512, 509, 485, 507] payload_rows_per_dst=[5560, 5590, 5558, 5598, 5626, 5422, 5592, 5480] dup_rows_per_dst=[2648, 2576, 2664, 2716, 2654, 2568, 2590, 2694] combine_rows_per_src=[5560, 5590, 5558, 5598, 5626, 5422, 5592, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=228736 combine_payload_bytes=159629312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159858048 ingress_bytes=184795136 link_bytes=184795136 token_bytes=319487360 weight_movement_bytes=25165824 total_nvlink_bytes=344653184 weight_to_token_ratio=0.078769 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406077466e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44532 payload_rows_remote=39032 dup_rows=21004 dup_rows_remote=18448 dedup_ratio=0.320496 router_tokens_per_expert=[476, 536, 512, 492, 564, 532, 545, 488, 531, 527, 501, 497, 515, 516, 554, 481, 538, 534, 566, 493, 521, 508, 523, 535, 525, 483, 525, 520, 517, 500, 445, 469, 483, 498, 472, 501, 525, 537, 494, 516, 509, 527, 472, 497, 495, 519, 512, 497, 500, 533, 518, 541, 482, 497, 493, 486, 520, 522, 537, 547, 534, 493, 501, 541] payload_rows_per_dst=[5594, 5630, 5678, 5470, 5510, 5500, 5520, 5630] dup_rows_per_dst=[2696, 2614, 2758, 2498, 2542, 2556, 2580, 2760] combine_rows_per_src=[5594, 5630, 5678, 5470, 5510, 5500, 5520, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=39018 dup_rows=20978 dup_rows_remote=18396 dedup_ratio=0.320099 router_tokens_per_expert=[530, 503, 498, 534, 532, 523, 518, 509, 544, 505, 499, 509, 517, 516, 498, 473, 536, 486, 540, 493, 482, 501, 512, 502, 517, 509, 517, 552, 520, 480, 509, 505, 524, 507, 495, 529, 486, 512, 466, 495, 555, 500, 541, 508, 490, 529, 503, 501, 511, 502, 477, 493, 520, 533, 513, 514, 491, 495, 543, 534, 543, 549, 518, 522] payload_rows_per_dst=[5526, 5540, 5614, 5606, 5534, 5584, 5484, 5670] dup_rows_per_dst=[2768, 2582, 2490, 2612, 2494, 2670, 2642, 2720] combine_rows_per_src=[5526, 5540, 5614, 5606, 5534, 5584, 5484, 5670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159039488 dispatch_meta_bytes=228888 combine_payload_bytes=159039488 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159268376 ingress_bytes=259702784 link_bytes=259702784 token_bytes=318307864 weight_movement_bytes=100663296 total_nvlink_bytes=418971160 weight_to_token_ratio=0.316245 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.976038116e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44410 payload_rows_remote=38620 dup_rows=21126 dup_rows_remote=18230 dedup_ratio=0.322357 router_tokens_per_expert=[484, 494, 499, 542, 508, 495, 528, 527, 511, 485, 515, 515, 521, 495, 535, 484, 526, 545, 543, 522, 506, 517, 533, 501, 518, 518, 516, 501, 514, 535, 532, 511, 505, 512, 519, 515, 512, 520, 527, 522, 517, 518, 517, 534, 483, 525, 459, 490, 523, 515, 490, 503, 536, 501, 509, 484, 503, 519, 509, 437, 487, 529, 550, 522] payload_rows_per_dst=[5790, 5510, 5514, 5528, 5578, 5532, 5500, 5458] dup_rows_per_dst=[2896, 2768, 2542, 2576, 2696, 2554, 2622, 2472] combine_rows_per_src=[5790, 5510, 5514, 5528, 5578, 5532, 5500, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44600 payload_rows_remote=38982 dup_rows=20936 dup_rows_remote=18474 dedup_ratio=0.319458 router_tokens_per_expert=[528, 502, 507, 529, 479, 507, 478, 484, 518, 499, 544, 492, 509, 511, 547, 472, 532, 543, 476, 498, 472, 505, 518, 496, 513, 521, 535, 506, 515, 466, 493, 521, 526, 539, 519, 519, 495, 507, 497, 532, 528, 534, 528, 511, 488, 527, 498, 535, 526, 471, 530, 522, 514, 514, 508, 547, 500, 511, 502, 497, 515, 528, 536, 548] payload_rows_per_dst=[5512, 5536, 5618, 5496, 5664, 5638, 5596, 5540] dup_rows_per_dst=[2516, 2648, 2462, 2644, 2604, 2660, 2668, 2734] combine_rows_per_src=[5512, 5536, 5618, 5496, 5664, 5638, 5596, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159875072 dispatch_meta_bytes=229920 combine_payload_bytes=159875072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160104992 ingress_bytes=185040896 link_bytes=185040896 token_bytes=319980064 weight_movement_bytes=25165824 total_nvlink_bytes=345145888 weight_to_token_ratio=0.078648 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407947416e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44610 payload_rows_remote=38972 dup_rows=20926 dup_rows_remote=18260 dedup_ratio=0.319305 router_tokens_per_expert=[494, 498, 478, 499, 467, 501, 459, 519, 527, 523, 548, 477, 540, 526, 516, 507, 510, 539, 532, 527, 525, 484, 520, 502, 481, 542, 476, 494, 485, 516, 536, 522, 509, 495, 527, 508, 513, 481, 527, 484, 524, 522, 566, 527, 493, 522, 512, 522, 527, 529, 531, 489, 529, 578, 488, 481, 536, 543, 501, 521, 513, 517, 507, 476] payload_rows_per_dst=[5408, 5628, 5592, 5554, 5504, 5674, 5638, 5612] dup_rows_per_dst=[2422, 2700, 2686, 2550, 2584, 2702, 2666, 2616] combine_rows_per_src=[5408, 5628, 5592, 5554, 5504, 5674, 5638, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=229656 combine_payload_bytes=159817728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160047384 ingress_bytes=184983552 link_bytes=184983552 token_bytes=319865112 weight_movement_bytes=25165824 total_nvlink_bytes=345030936 weight_to_token_ratio=0.078676 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407511094e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44318 payload_rows_remote=38802 dup_rows=21218 dup_rows_remote=18624 dedup_ratio=0.323761 router_tokens_per_expert=[525, 498, 504, 528, 500, 501, 529, 476, 505, 539, 515, 540, 502, 494, 510, 492, 509, 508, 515, 516, 496, 483, 517, 538, 506, 511, 476, 521, 547, 535, 551, 517, 519, 492, 522, 475, 548, 472, 539, 520, 523, 508, 546, 523, 489, 528, 480, 497, 526, 493, 504, 551, 490, 525, 502, 546, 485, 520, 514, 513, 499, 504, 503, 508] payload_rows_per_dst=[5496, 5438, 5520, 5628, 5532, 5568, 5620, 5516] dup_rows_per_dst=[2588, 2526, 2644, 2878, 2642, 2602, 2744, 2594] combine_rows_per_src=[5496, 5438, 5520, 5628, 5532, 5568, 5620, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44280 payload_rows_remote=38628 dup_rows=21256 dup_rows_remote=18456 dedup_ratio=0.324341 router_tokens_per_expert=[511, 500, 533, 535, 517, 530, 485, 521, 489, 504, 488, 534, 514, 482, 468, 512, 511, 517, 531, 523, 521, 541, 527, 507, 461, 505, 516, 547, 555, 508, 537, 499, 529, 532, 543, 546, 539, 508, 517, 512, 485, 511, 507, 472, 559, 459, 523, 483, 522, 485, 523, 521, 503, 495, 508, 509, 537, 474, 496, 492, 517, 480, 522, 530] payload_rows_per_dst=[5588, 5428, 5562, 5560, 5652, 5506, 5524, 5460] dup_rows_per_dst=[2676, 2554, 2794, 2696, 2800, 2492, 2608, 2636] combine_rows_per_src=[5588, 5428, 5562, 5560, 5652, 5506, 5524, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158187520 dispatch_meta_bytes=227400 combine_payload_bytes=158187520 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158414920 ingress_bytes=183353344 link_bytes=183353344 token_bytes=316602440 weight_movement_bytes=25165824 total_nvlink_bytes=341768264 weight_to_token_ratio=0.079487 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.395107095e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229824 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159900096 ingress_bytes=210001920 link_bytes=210001920 token_bytes=319570368 weight_movement_bytes=50331648 total_nvlink_bytes=369902016 weight_to_token_ratio=0.157498 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597871967e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=228928 combine_payload_bytes=159629312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159858240 ingress_bytes=184795136 link_bytes=184795136 token_bytes=319487552 weight_movement_bytes=25165824 total_nvlink_bytes=344653376 weight_to_token_ratio=0.078769 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406077466e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158932992 dispatch_meta_bytes=229704 combine_payload_bytes=158932992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159162696 ingress_bytes=184098816 link_bytes=184098816 token_bytes=318095688 weight_movement_bytes=25165824 total_nvlink_bytes=343261512 weight_to_token_ratio=0.079114 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400779275e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158220288 dispatch_meta_bytes=228336 combine_payload_bytes=158220288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158448624 ingress_bytes=183386112 link_bytes=183386112 token_bytes=316668912 weight_movement_bytes=25165824 total_nvlink_bytes=341834736 weight_to_token_ratio=0.079470 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.395356421e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44366 payload_rows_remote=38776 dup_rows=21170 dup_rows_remote=18504 dedup_ratio=0.323029 router_tokens_per_expert=[540, 506, 516, 542, 522, 526, 519, 515, 510, 497, 519, 493, 453, 535, 526, 509, 501, 450, 533, 514, 499, 502, 516, 502, 518, 522, 512, 486, 537, 529, 489, 535, 515, 509, 517, 501, 482, 533, 566, 480, 514, 514, 540, 477, 501, 516, 484, 543, 513, 493, 494, 526, 514, 523, 542, 475, 524, 497, 513, 482, 560, 531, 507, 509] payload_rows_per_dst=[5586, 5602, 5526, 5590, 5516, 5482, 5520, 5544] dup_rows_per_dst=[2786, 2482, 2508, 2666, 2690, 2696, 2640, 2702] combine_rows_per_src=[5586, 5602, 5526, 5590, 5516, 5482, 5520, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44296 payload_rows_remote=38674 dup_rows=21240 dup_rows_remote=18568 dedup_ratio=0.324097 router_tokens_per_expert=[551, 526, 479, 487, 520, 495, 496, 549, 516, 509, 516, 514, 491, 525, 530, 502, 571, 494, 495, 506, 510, 489, 510, 537, 509, 523, 491, 469, 513, 533, 526, 513, 508, 552, 506, 467, 534, 529, 529, 478, 497, 527, 517, 526, 548, 518, 499, 515, 537, 523, 501, 480, 488, 484, 541, 534, 514, 516, 491, 468, 529, 492, 497, 528] payload_rows_per_dst=[5584, 5498, 5544, 5456, 5576, 5622, 5560, 5456] dup_rows_per_dst=[2622, 2708, 2680, 2698, 2630, 2672, 2616, 2614] combine_rows_per_src=[5584, 5498, 5544, 5456, 5576, 5622, 5560, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44368 payload_rows_remote=38768 dup_rows=21168 dup_rows_remote=18494 dedup_ratio=0.322998 router_tokens_per_expert=[495, 525, 494, 489, 509, 521, 513, 537, 482, 537, 527, 530, 502, 535, 493, 531, 484, 487, 527, 532, 522, 518, 529, 519, 493, 517, 502, 498, 529, 487, 522, 538, 524, 511, 508, 507, 508, 500, 497, 476, 541, 549, 512, 501, 492, 537, 500, 527, 449, 543, 513, 514, 468, 524, 494, 524, 516, 505, 501, 491, 553, 532, 496, 531] payload_rows_per_dst=[5530, 5600, 5528, 5568, 5522, 5602, 5440, 5578] dup_rows_per_dst=[2636, 2674, 2708, 2604, 2540, 2716, 2618, 2672] combine_rows_per_src=[5530, 5600, 5528, 5568, 5522, 5602, 5440, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=229120 combine_payload_bytes=158826496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159055616 ingress_bytes=183992320 link_bytes=183992320 token_bytes=317882112 weight_movement_bytes=25165824 total_nvlink_bytes=343047936 weight_to_token_ratio=0.079167 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399968964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44400 payload_rows_remote=38718 dup_rows=21136 dup_rows_remote=18412 dedup_ratio=0.322510 router_tokens_per_expert=[493, 546, 562, 533, 521, 474, 549, 520, 530, 516, 506, 528, 521, 485, 486, 561, 495, 535, 518, 499, 525, 468, 517, 508, 498, 457, 482, 518, 517, 511, 492, 516, 518, 547, 514, 535, 487, 510, 508, 508, 518, 536, 518, 536, 507, 511, 498, 498, 504, 497, 512, 546, 498, 480, 511, 522, 537, 472, 492, 494, 506, 512, 532, 517] payload_rows_per_dst=[5682, 5558, 5732, 5406, 5444, 5562, 5486, 5530] dup_rows_per_dst=[2724, 2708, 2850, 2576, 2506, 2672, 2506, 2594] combine_rows_per_src=[5682, 5558, 5732, 5406, 5444, 5562, 5486, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44610 payload_rows_remote=39046 dup_rows=20926 dup_rows_remote=18372 dedup_ratio=0.319305 router_tokens_per_expert=[491, 520, 527, 527, 498, 468, 492, 516, 530, 500, 522, 518, 482, 550, 540, 513, 501, 530, 494, 493, 508, 519, 494, 520, 476, 486, 515, 487, 534, 487, 526, 489, 521, 533, 519, 503, 513, 524, 485, 522, 520, 551, 537, 481, 520, 519, 486, 488, 502, 513, 499, 531, 537, 541, 516, 548, 516, 516, 547, 484, 519, 502, 540, 482] payload_rows_per_dst=[5486, 5600, 5564, 5510, 5596, 5574, 5670, 5610] dup_rows_per_dst=[2592, 2710, 2554, 2490, 2644, 2630, 2704, 2602] combine_rows_per_src=[5486, 5600, 5564, 5510, 5596, 5574, 5670, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158408704 dispatch_meta_bytes=228968 combine_payload_bytes=158408704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158637672 ingress_bytes=183574528 link_bytes=183574528 token_bytes=317046376 weight_movement_bytes=25165824 total_nvlink_bytes=342212200 weight_to_token_ratio=0.079376 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396790049e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38956 dup_rows=21102 dup_rows_remote=18404 dedup_ratio=0.321991 router_tokens_per_expert=[518, 490, 517, 479, 533, 513, 493, 541, 539, 507, 497, 502, 501, 524, 543, 483, 535, 490, 498, 484, 476, 504, 573, 531, 521, 526, 517, 508, 511, 550, 493, 558, 511, 501, 484, 533, 524, 497, 514, 544, 503, 486, 482, 461, 519, 522, 491, 482, 492, 504, 497, 501, 534, 491, 555, 514, 510, 562, 509, 527, 524, 529, 491, 519] payload_rows_per_dst=[5562, 5504, 5558, 5634, 5646, 5442, 5478, 5610] dup_rows_per_dst=[2606, 2688, 2624, 2734, 2570, 2450, 2698, 2732] combine_rows_per_src=[5562, 5504, 5558, 5634, 5646, 5442, 5478, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158793728 dispatch_meta_bytes=229048 combine_payload_bytes=158793728 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159022776 ingress_bytes=234291200 link_bytes=234291200 token_bytes=317816504 weight_movement_bytes=75497472 total_nvlink_bytes=393313976 weight_to_token_ratio=0.237551 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782685323e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44358 payload_rows_remote=38766 dup_rows=21178 dup_rows_remote=18386 dedup_ratio=0.323151 router_tokens_per_expert=[490, 532, 531, 504, 552, 498, 514, 523, 529, 523, 507, 522, 518, 522, 538, 486, 524, 547, 521, 484, 517, 516, 537, 537, 524, 480, 514, 483, 509, 493, 473, 490, 528, 505, 471, 514, 463, 525, 490, 452, 503, 563, 545, 483, 496, 508, 513, 495, 499, 521, 522, 528, 507, 495, 489, 523, 509, 529, 488, 568, 535, 509, 528, 526] payload_rows_per_dst=[5400, 5292, 5522, 5720, 5578, 5590, 5664, 5592] dup_rows_per_dst=[2388, 2368, 2576, 2842, 2818, 2622, 2772, 2792] combine_rows_per_src=[5400, 5292, 5522, 5720, 5578, 5590, 5664, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=38896 dup_rows=20988 dup_rows_remote=18256 dedup_ratio=0.320251 router_tokens_per_expert=[478, 450, 530, 489, 516, 499, 495, 529, 532, 505, 545, 505, 510, 519, 513, 465, 485, 539, 519, 528, 505, 506, 465, 479, 510, 535, 514, 556, 537, 487, 476, 515, 549, 514, 496, 519, 501, 509, 566, 538, 536, 508, 490, 498, 505, 529, 534, 517, 524, 484, 551, 511, 492, 530, 539, 510, 530, 504, 514, 499, 498, 503, 540, 494] payload_rows_per_dst=[5470, 5604, 5530, 5504, 5652, 5634, 5642, 5512] dup_rows_per_dst=[2502, 2584, 2522, 2756, 2732, 2600, 2640, 2652] combine_rows_per_src=[5470, 5604, 5530, 5504, 5652, 5634, 5642, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158588928 dispatch_meta_bytes=228520 combine_payload_bytes=158588928 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158817448 ingress_bytes=208920576 link_bytes=208920576 token_bytes=317406376 weight_movement_bytes=50331648 total_nvlink_bytes=367738024 weight_to_token_ratio=0.158572 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589644189e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159932416 dispatch_meta_bytes=229672 combine_payload_bytes=159932416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160162088 ingress_bytes=185098240 link_bytes=185098240 token_bytes=320094504 weight_movement_bytes=25165824 total_nvlink_bytes=345260328 weight_to_token_ratio=0.078620 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408383737e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=229440 combine_payload_bytes=159563776 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159793216 ingress_bytes=209895424 link_bytes=209895424 token_bytes=319356992 weight_movement_bytes=50331648 total_nvlink_bytes=369688640 weight_to_token_ratio=0.157603 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597061656e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=39030 dup_rows=21036 dup_rows_remote=18490 dedup_ratio=0.320984 router_tokens_per_expert=[508, 504, 547, 521, 526, 508, 535, 540, 487, 524, 538, 532, 493, 525, 579, 555, 518, 449, 498, 540, 540, 499, 523, 540, 502, 474, 498, 478, 545, 518, 489, 504, 537, 505, 485, 514, 486, 494, 536, 507, 520, 506, 503, 523, 517, 546, 483, 510, 470, 507, 542, 515, 499, 514, 530, 500, 499, 475, 512, 483, 504, 514, 502, 493] payload_rows_per_dst=[5674, 5664, 5622, 5470, 5524, 5556, 5540, 5450] dup_rows_per_dst=[2704, 2802, 2592, 2546, 2604, 2660, 2614, 2514] combine_rows_per_src=[5674, 5664, 5622, 5470, 5524, 5556, 5540, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158785536 dispatch_meta_bytes=228608 combine_payload_bytes=158785536 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159014144 ingress_bytes=209117184 link_bytes=209117184 token_bytes=317799680 weight_movement_bytes=50331648 total_nvlink_bytes=368131328 weight_to_token_ratio=0.158375 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591140149e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159318016 dispatch_meta_bytes=228608 combine_payload_bytes=159318016 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159546624 ingress_bytes=209649664 link_bytes=209649664 token_bytes=318864640 weight_movement_bytes=50331648 total_nvlink_bytes=369196288 weight_to_token_ratio=0.157846 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595191706e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44690 payload_rows_remote=39084 dup_rows=20846 dup_rows_remote=18146 dedup_ratio=0.318085 router_tokens_per_expert=[508, 551, 533, 500, 519, 500, 526, 508, 532, 500, 487, 547, 521, 500, 517, 525, 541, 492, 497, 545, 540, 490, 485, 494, 503, 527, 468, 498, 498, 486, 488, 528, 497, 522, 513, 536, 530, 503, 510, 541, 500, 513, 540, 529, 529, 498, 502, 542, 530, 505, 513, 497, 503, 466, 523, 528, 477, 488, 481, 543, 511, 506, 515, 523] payload_rows_per_dst=[5684, 5574, 5526, 5488, 5628, 5606, 5580, 5604] dup_rows_per_dst=[2606, 2684, 2642, 2504, 2676, 2700, 2550, 2484] combine_rows_per_src=[5684, 5574, 5526, 5488, 5628, 5606, 5580, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44376 payload_rows_remote=38826 dup_rows=21160 dup_rows_remote=18542 dedup_ratio=0.322876 router_tokens_per_expert=[510, 511, 515, 536, 515, 502, 534, 515, 523, 477, 535, 523, 487, 510, 513, 516, 519, 537, 525, 491, 514, 482, 555, 553, 502, 522, 531, 505, 515, 511, 499, 516, 529, 492, 449, 508, 462, 527, 526, 509, 527, 459, 481, 525, 511, 523, 535, 516, 499, 528, 493, 504, 522, 509, 553, 528, 533, 512, 511, 489, 533, 487, 507, 482] payload_rows_per_dst=[5540, 5550, 5640, 5552, 5490, 5556, 5546, 5502] dup_rows_per_dst=[2736, 2618, 2712, 2650, 2514, 2598, 2726, 2606] combine_rows_per_src=[5540, 5550, 5640, 5552, 5490, 5556, 5546, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=230080 combine_payload_bytes=159866880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160096960 ingress_bytes=185032704 link_bytes=185032704 token_bytes=319963840 weight_movement_bytes=25165824 total_nvlink_bytes=345129664 weight_to_token_ratio=0.078652 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407885084e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38780 dup_rows=21146 dup_rows_remote=18404 dedup_ratio=0.322662 router_tokens_per_expert=[530, 521, 494, 513, 483, 541, 523, 515, 535, 533, 487, 496, 497, 513, 507, 512, 547, 504, 512, 524, 553, 489, 566, 481, 520, 467, 539, 504, 516, 498, 528, 511, 488, 548, 561, 503, 487, 484, 513, 491, 519, 521, 511, 520, 506, 493, 477, 515, 500, 522, 514, 513, 550, 503, 528, 485, 483, 520, 520, 486, 523, 540, 475, 510] payload_rows_per_dst=[5624, 5478, 5610, 5588, 5508, 5506, 5518, 5558] dup_rows_per_dst=[2616, 2682, 2742, 2578, 2642, 2618, 2712, 2556] combine_rows_per_src=[5624, 5478, 5610, 5588, 5508, 5506, 5518, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44240 payload_rows_remote=38662 dup_rows=21296 dup_rows_remote=18564 dedup_ratio=0.324951 router_tokens_per_expert=[512, 535, 505, 483, 536, 504, 498, 535, 528, 514, 487, 528, 511, 502, 520, 501, 534, 500, 496, 532, 511, 519, 488, 479, 493, 485, 529, 485, 541, 530, 499, 513, 520, 479, 541, 507, 546, 495, 499, 495, 491, 543, 533, 545, 507, 506, 525, 522, 537, 510, 457, 534, 524, 507, 487, 512, 533, 528, 516, 479, 515, 528, 520, 494] payload_rows_per_dst=[5578, 5534, 5472, 5470, 5574, 5576, 5532, 5504] dup_rows_per_dst=[2732, 2778, 2518, 2680, 2590, 2718, 2604, 2676] combine_rows_per_src=[5578, 5534, 5472, 5470, 5574, 5576, 5532, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44288 payload_rows_remote=38844 dup_rows=21248 dup_rows_remote=18634 dedup_ratio=0.324219 router_tokens_per_expert=[520, 526, 506, 523, 503, 531, 521, 496, 477, 485, 561, 499, 526, 498, 558, 520, 545, 490, 512, 528, 496, 508, 535, 503, 481, 527, 524, 506, 504, 510, 509, 483, 539, 478, 511, 500, 481, 541, 507, 531, 525, 523, 510, 514, 551, 520, 481, 487, 499, 503, 518, 489, 490, 479, 547, 504, 499, 557, 470, 530, 498, 517, 517, 541] payload_rows_per_dst=[5560, 5584, 5582, 5510, 5488, 5536, 5444, 5584] dup_rows_per_dst=[2692, 2664, 2652, 2578, 2688, 2686, 2614, 2674] combine_rows_per_src=[5560, 5584, 5582, 5510, 5488, 5536, 5444, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=160088064 dispatch_meta_bytes=228920 combine_payload_bytes=160088064 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160316984 ingress_bytes=210419712 link_bytes=210419712 token_bytes=320405048 weight_movement_bytes=50331648 total_nvlink_bytes=370736696 weight_to_token_ratio=0.157088 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601050882e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159031296 dispatch_meta_bytes=229472 combine_payload_bytes=159031296 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159260768 ingress_bytes=209362944 link_bytes=209362944 token_bytes=318292064 weight_movement_bytes=50331648 total_nvlink_bytes=368623712 weight_to_token_ratio=0.158130 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593010098e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44720 payload_rows_remote=39142 dup_rows=20816 dup_rows_remote=18266 dedup_ratio=0.317627 router_tokens_per_expert=[511, 552, 532, 489, 514, 521, 488, 513, 506, 525, 542, 494, 489, 514, 514, 511, 547, 504, 524, 542, 544, 499, 484, 516, 516, 483, 486, 529, 511, 499, 521, 476, 522, 496, 526, 515, 500, 517, 531, 537, 498, 504, 522, 517, 507, 512, 511, 467, 537, 494, 512, 523, 502, 519, 582, 519, 527, 490, 498, 468, 513, 502, 511, 493] payload_rows_per_dst=[5438, 5588, 5688, 5594, 5602, 5518, 5714, 5578] dup_rows_per_dst=[2430, 2602, 2768, 2652, 2686, 2436, 2692, 2550] combine_rows_per_src=[5438, 5588, 5688, 5594, 5602, 5518, 5714, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44324 payload_rows_remote=38776 dup_rows=21212 dup_rows_remote=18464 dedup_ratio=0.323669 router_tokens_per_expert=[498, 482, 522, 516, 510, 541, 500, 496, 540, 528, 501, 505, 509, 528, 535, 494, 502, 521, 538, 480, 503, 543, 518, 537, 470, 523, 524, 508, 516, 498, 488, 522, 496, 493, 471, 558, 550, 524, 545, 511, 485, 512, 511, 513, 508, 486, 488, 493, 482, 508, 524, 477, 527, 536, 513, 503, 456, 548, 523, 513, 540, 515, 534, 529] payload_rows_per_dst=[5536, 5604, 5612, 5486, 5548, 5404, 5544, 5590] dup_rows_per_dst=[2594, 2676, 2672, 2612, 2748, 2588, 2596, 2726] combine_rows_per_src=[5536, 5604, 5612, 5486, 5548, 5404, 5544, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=228736 combine_payload_bytes=158842880 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159071616 ingress_bytes=234340352 link_bytes=234340352 token_bytes=317914496 weight_movement_bytes=75497472 total_nvlink_bytes=393411968 weight_to_token_ratio=0.237477 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783059313e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158359552 dispatch_meta_bytes=228904 combine_payload_bytes=158359552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158588456 ingress_bytes=208691200 link_bytes=208691200 token_bytes=316948008 weight_movement_bytes=50331648 total_nvlink_bytes=367279656 weight_to_token_ratio=0.158801 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.587898903e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=229912 combine_payload_bytes=159105024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159334936 ingress_bytes=184270848 link_bytes=184270848 token_bytes=318439960 weight_movement_bytes=25165824 total_nvlink_bytes=343605784 weight_to_token_ratio=0.079028 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402088240e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=38896 dup_rows=21086 dup_rows_remote=18468 dedup_ratio=0.321747 router_tokens_per_expert=[515, 486, 525, 565, 511, 504, 505, 553, 543, 538, 512, 513, 510, 501, 489, 523, 548, 507, 486, 514, 516, 477, 517, 530, 525, 528, 509, 539, 512, 472, 500, 501, 529, 481, 540, 511, 511, 542, 503, 519, 483, 492, 521, 508, 513, 476, 495, 555, 498, 484, 518, 518, 480, 487, 516, 516, 540, 515, 502, 478, 546, 520, 527, 470] payload_rows_per_dst=[5636, 5518, 5576, 5554, 5590, 5512, 5480, 5584] dup_rows_per_dst=[2692, 2740, 2614, 2618, 2682, 2574, 2554, 2612] combine_rows_per_src=[5636, 5518, 5576, 5554, 5590, 5512, 5480, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160325632 dispatch_meta_bytes=229632 combine_payload_bytes=160325632 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160555264 ingress_bytes=185491456 link_bytes=185491456 token_bytes=320880896 weight_movement_bytes=25165824 total_nvlink_bytes=346046720 weight_to_token_ratio=0.078427 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411375657e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=228960 combine_payload_bytes=158826496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159055456 ingress_bytes=183992320 link_bytes=183992320 token_bytes=317881952 weight_movement_bytes=25165824 total_nvlink_bytes=343047776 weight_to_token_ratio=0.079167 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399968964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44704 payload_rows_remote=39104 dup_rows=20832 dup_rows_remote=18110 dedup_ratio=0.317871 router_tokens_per_expert=[502, 536, 535, 487, 492, 522, 497, 510, 508, 494, 546, 535, 546, 505, 496, 491, 533, 492, 510, 469, 505, 506, 496, 495, 540, 488, 528, 513, 500, 491, 512, 555, 505, 483, 496, 522, 517, 542, 523, 523, 558, 517, 529, 511, 523, 505, 517, 501, 541, 480, 491, 531, 509, 506, 501, 496, 532, 482, 525, 520, 514, 513, 494, 526] payload_rows_per_dst=[5592, 5664, 5484, 5692, 5570, 5600, 5514, 5588] dup_rows_per_dst=[2570, 2578, 2528, 2562, 2652, 2722, 2596, 2624] combine_rows_per_src=[5592, 5664, 5484, 5692, 5570, 5600, 5514, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44360 payload_rows_remote=38780 dup_rows=21176 dup_rows_remote=18414 dedup_ratio=0.323120 router_tokens_per_expert=[455, 535, 490, 541, 515, 525, 516, 503, 516, 510, 515, 525, 548, 513, 516, 528, 537, 542, 490, 490, 514, 511, 519, 452, 543, 504, 518, 505, 543, 537, 517, 523, 517, 498, 509, 489, 476, 504, 534, 471, 534, 506, 517, 501, 528, 535, 505, 543, 531, 530, 500, 511, 506, 477, 539, 503, 526, 468, 508, 491, 492, 480, 509, 534] payload_rows_per_dst=[5536, 5580, 5560, 5700, 5454, 5558, 5508, 5464] dup_rows_per_dst=[2624, 2762, 2550, 2680, 2542, 2780, 2686, 2552] combine_rows_per_src=[5536, 5580, 5560, 5700, 5454, 5558, 5508, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159318016 dispatch_meta_bytes=229456 combine_payload_bytes=159318016 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159547472 ingress_bytes=184483840 link_bytes=184483840 token_bytes=318865488 weight_movement_bytes=25165824 total_nvlink_bytes=344031312 weight_to_token_ratio=0.078923 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403708863e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5100480920 prefetch_bytes=276824064 reduce_bytes=503316480 weight_movement_bytes=780140544 egress_bytes=2552072600 ingress_bytes=3328548864 link_bytes=3328548864 total_nvlink_bytes=5880621464 weight_to_token_ratio=0.152954 prefetch_share_of_ingress=0.083167 dedup_ratio=0.321297 prefetch_slots=22 unused_prefetch_slots=234 slot_utilization=0.085938 reduce_slots=20 remote_demand=11 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=19685 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.582900683e-04 prefetch_bytes_per_gemm_flop=1.316444547e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44618 payload_rows_remote=39000 dup_rows=20918 dup_rows_remote=18236 dedup_ratio=0.319183 router_tokens_per_expert=[487, 512, 487, 532, 503, 486, 513, 511, 509, 521, 481, 511, 506, 529, 516, 528, 536, 546, 504, 521, 525, 490, 515, 513, 513, 505, 489, 511, 495, 473, 518, 537, 513, 475, 495, 495, 487, 512, 533, 516, 512, 515, 520, 519, 465, 527, 527, 507, 530, 526, 514, 482, 539, 540, 511, 529, 531, 491, 502, 525, 535, 539, 530, 503] payload_rows_per_dst=[5534, 5562, 5618, 5546, 5506, 5584, 5656, 5612] dup_rows_per_dst=[2528, 2640, 2682, 2536, 2546, 2600, 2686, 2700] combine_rows_per_src=[5534, 5562, 5618, 5546, 5506, 5584, 5656, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44632 payload_rows_remote=38996 dup_rows=20904 dup_rows_remote=18282 dedup_ratio=0.318970 router_tokens_per_expert=[545, 531, 501, 503, 538, 502, 505, 504, 501, 484, 556, 505, 471, 538, 533, 485, 535, 517, 509, 482, 511, 496, 513, 506, 490, 507, 516, 534, 520, 524, 501, 528, 555, 520, 503, 492, 517, 550, 486, 503, 460, 517, 529, 541, 525, 511, 468, 538, 508, 491, 487, 517, 503, 561, 504, 510, 517, 499, 504, 501, 517, 491, 542, 510] payload_rows_per_dst=[5636, 5628, 5614, 5554, 5560, 5506, 5592, 5542] dup_rows_per_dst=[2622, 2660, 2524, 2686, 2714, 2508, 2570, 2620] combine_rows_per_src=[5636, 5628, 5614, 5554, 5560, 5506, 5592, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160169984 dispatch_meta_bytes=228856 combine_payload_bytes=160169984 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160398840 ingress_bytes=210501632 link_bytes=210501632 token_bytes=320568824 weight_movement_bytes=50331648 total_nvlink_bytes=370900472 weight_to_token_ratio=0.157007 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601674198e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=228776 combine_payload_bytes=158842880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159071656 ingress_bytes=184008704 link_bytes=184008704 token_bytes=317914536 weight_movement_bytes=25165824 total_nvlink_bytes=343080360 weight_to_token_ratio=0.079159 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400093627e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44606 payload_rows_remote=39116 dup_rows=20930 dup_rows_remote=18320 dedup_ratio=0.319366 router_tokens_per_expert=[526, 521, 481, 510, 508, 475, 520, 552, 509, 491, 517, 521, 544, 496, 563, 497, 506, 507, 497, 508, 486, 520, 511, 553, 504, 506, 475, 510, 528, 504, 520, 506, 500, 513, 501, 514, 508, 522, 455, 507, 518, 535, 529, 532, 517, 540, 556, 517, 480, 518, 540, 502, 484, 497, 488, 541, 529, 488, 506, 524, 510, 490, 515, 520] payload_rows_per_dst=[5536, 5612, 5572, 5584, 5516, 5710, 5490, 5586] dup_rows_per_dst=[2650, 2664, 2604, 2522, 2524, 2778, 2610, 2578] combine_rows_per_src=[5536, 5612, 5572, 5584, 5516, 5710, 5490, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5103714640 prefetch_bytes=377487360 reduce_bytes=327155712 weight_movement_bytes=704643072 egress_bytes=2553692496 ingress_bytes=3254665216 link_bytes=3254665216 total_nvlink_bytes=5808357712 weight_to_token_ratio=0.138065 prefetch_share_of_ingress=0.115983 dedup_ratio=0.321266 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=13 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.547765108e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38850 dup_rows=21092 dup_rows_remote=18494 dedup_ratio=0.321838 router_tokens_per_expert=[505, 505, 505, 504, 558, 542, 509, 562, 513, 500, 491, 486, 492, 511, 492, 481, 477, 531, 548, 515, 506, 487, 487, 512, 517, 538, 522, 504, 484, 520, 483, 524, 511, 528, 525, 505, 533, 517, 516, 461, 493, 523, 499, 529, 537, 528, 541, 472, 497, 498, 503, 468, 506, 498, 527, 529, 552, 555, 472, 545, 538, 516, 500, 535] payload_rows_per_dst=[5616, 5448, 5554, 5478, 5594, 5616, 5446, 5692] dup_rows_per_dst=[2764, 2484, 2572, 2706, 2598, 2628, 2606, 2734] combine_rows_per_src=[5616, 5448, 5554, 5478, 5594, 5616, 5446, 5692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159744000 dispatch_meta_bytes=228944 combine_payload_bytes=159744000 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159972944 ingress_bytes=184909824 link_bytes=184909824 token_bytes=319716944 weight_movement_bytes=25165824 total_nvlink_bytes=344882768 weight_to_token_ratio=0.078713 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406950109e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159727616 dispatch_meta_bytes=229112 combine_payload_bytes=159727616 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159956728 ingress_bytes=260390912 link_bytes=260390912 token_bytes=319684344 weight_movement_bytes=100663296 total_nvlink_bytes=420347640 weight_to_token_ratio=0.314883 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.981273975e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5104318472 prefetch_bytes=327155712 reduce_bytes=478150656 weight_movement_bytes=805306368 egress_bytes=2553993224 ingress_bytes=3355631616 link_bytes=3355861512 total_nvlink_bytes=5909624840 weight_to_token_ratio=0.157770 prefetch_share_of_ingress=0.097495 dedup_ratio=0.321188 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=19 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.595889289e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160219136 dispatch_meta_bytes=229744 combine_payload_bytes=160219136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160448880 ingress_bytes=185384960 link_bytes=185384960 token_bytes=320668016 weight_movement_bytes=25165824 total_nvlink_bytes=345833840 weight_to_token_ratio=0.078479 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410565345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159129600 dispatch_meta_bytes=229376 combine_payload_bytes=159129600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159358976 ingress_bytes=184295424 link_bytes=184295424 token_bytes=318488576 weight_movement_bytes=25165824 total_nvlink_bytes=343654400 weight_to_token_ratio=0.079016 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402275235e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5103615096 prefetch_bytes=377487360 reduce_bytes=301989888 weight_movement_bytes=679477248 egress_bytes=2553642104 ingress_bytes=3229450240 link_bytes=3229450240 total_nvlink_bytes=5783092344 weight_to_token_ratio=0.133136 prefetch_share_of_ingress=0.116889 dedup_ratio=0.321383 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=12 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.535774056e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44544 payload_rows_remote=38992 dup_rows=20992 dup_rows_remote=18316 dedup_ratio=0.320312 router_tokens_per_expert=[517, 519, 519, 556, 471, 495, 543, 527, 486, 506, 550, 487, 484, 514, 475, 522, 517, 490, 557, 497, 486, 503, 504, 466, 529, 505, 492, 502, 526, 500, 513, 496, 514, 534, 544, 517, 543, 531, 505, 523, 543, 486, 495, 558, 527, 502, 493, 510, 529, 531, 499, 487, 497, 537, 544, 499, 514, 517, 511, 537, 491, 492, 507, 497] payload_rows_per_dst=[5684, 5508, 5532, 5538, 5594, 5552, 5606, 5530] dup_rows_per_dst=[2610, 2540, 2508, 2588, 2828, 2676, 2640, 2602] combine_rows_per_src=[5684, 5508, 5532, 5538, 5594, 5552, 5606, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5091495880 prefetch_bytes=327155712 reduce_bytes=427819008 weight_movement_bytes=754974720 egress_bytes=2547576776 ingress_bytes=3298893824 link_bytes=3298893824 total_nvlink_bytes=5846470600 weight_to_token_ratio=0.148282 prefetch_share_of_ingress=0.099171 dedup_ratio=0.321690 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=17 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.568798146e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5106927152 prefetch_bytes=352321536 reduce_bytes=226492416 weight_movement_bytes=578813952 egress_bytes=2555299376 ingress_bytes=3130441728 link_bytes=3130441728 total_nvlink_bytes=5685741104 weight_to_token_ratio=0.113339 prefetch_share_of_ingress=0.112547 dedup_ratio=0.321304 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=9 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.488690283e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5100122920 prefetch_bytes=377487360 reduce_bytes=226492416 weight_movement_bytes=603979776 egress_bytes=2551894824 ingress_bytes=3152207872 link_bytes=3152207872 total_nvlink_bytes=5704102696 weight_to_token_ratio=0.118425 prefetch_share_of_ingress=0.119753 dedup_ratio=0.321533 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=9 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.499041233e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159711232 dispatch_meta_bytes=229232 combine_payload_bytes=159711232 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159940464 ingress_bytes=260374528 link_bytes=260374528 token_bytes=319651696 weight_movement_bytes=100663296 total_nvlink_bytes=420314992 weight_to_token_ratio=0.314916 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.981149312e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5106973288 prefetch_bytes=377487360 reduce_bytes=301989888 weight_movement_bytes=679477248 egress_bytes=2555320936 ingress_bytes=3231129600 link_bytes=3231129600 total_nvlink_bytes=5786450536 weight_to_token_ratio=0.133049 prefetch_share_of_ingress=0.116828 dedup_ratio=0.320908 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=12 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.536572681e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none + [2026-09-21 08:32:49] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 213.3 | throughput per GPU (TFLOP/s/GPU): 151.0 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.016327E+01 | loss scale: 1.0 | grad norm: 117.553 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=39040 dup_rows=20982 dup_rows_remote=18454 dedup_ratio=0.320160 router_tokens_per_expert=[504, 517, 534, 508, 536, 525, 536, 534, 568, 512, 505, 516, 492, 526, 509, 509, 503, 477, 481, 533, 503, 477, 523, 515, 469, 506, 535, 507, 524, 532, 478, 546, 497, 478, 526, 522, 492, 492, 511, 503, 536, 521, 541, 528, 479, 504, 485, 528, 484, 525, 528, 523, 494, 502, 536, 514, 508, 497, 513, 502, 511, 494, 539, 515] payload_rows_per_dst=[5620, 5620, 5482, 5598, 5514, 5582, 5588, 5550] dup_rows_per_dst=[2768, 2654, 2542, 2596, 2528, 2662, 2624, 2608] combine_rows_per_src=[5620, 5620, 5482, 5598, 5514, 5582, 5588, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=38966 dup_rows=21070 dup_rows_remote=18540 dedup_ratio=0.321503 router_tokens_per_expert=[509, 532, 496, 526, 485, 503, 522, 504, 512, 526, 544, 526, 537, 521, 498, 568, 516, 484, 490, 484, 481, 539, 520, 501, 529, 495, 494, 507, 555, 500, 502, 505, 514, 478, 489, 487, 486, 528, 534, 486, 508, 546, 534, 508, 522, 500, 477, 522, 535, 535, 501, 529, 467, 519, 564, 524, 468, 498, 503, 527, 509, 544, 495, 520] payload_rows_per_dst=[5594, 5642, 5500, 5572, 5480, 5598, 5576, 5504] dup_rows_per_dst=[2560, 2822, 2530, 2602, 2524, 2636, 2772, 2624] combine_rows_per_src=[5594, 5642, 5500, 5572, 5480, 5598, 5576, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44632 payload_rows_remote=39174 dup_rows=20904 dup_rows_remote=18352 dedup_ratio=0.318970 router_tokens_per_expert=[516, 529, 530, 532, 484, 542, 483, 508, 555, 507, 515, 538, 500, 514, 506, 502, 510, 482, 538, 505, 491, 518, 526, 504, 541, 530, 536, 514, 541, 506, 484, 478, 532, 519, 520, 518, 551, 499, 510, 512, 500, 460, 513, 523, 474, 503, 511, 521, 496, 495, 549, 494, 473, 500, 501, 497, 524, 550, 506, 531, 502, 512, 479, 528] payload_rows_per_dst=[5550, 5614, 5566, 5656, 5644, 5514, 5458, 5630] dup_rows_per_dst=[2698, 2660, 2582, 2604, 2678, 2496, 2552, 2634] combine_rows_per_src=[5550, 5614, 5566, 5656, 5644, 5514, 5458, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44494 payload_rows_remote=39016 dup_rows=21042 dup_rows_remote=18396 dedup_ratio=0.321075 router_tokens_per_expert=[517, 503, 507, 504, 535, 514, 485, 510, 493, 505, 525, 542, 516, 503, 482, 496, 523, 489, 501, 499, 512, 497, 494, 535, 546, 519, 537, 505, 522, 520, 496, 506, 514, 519, 518, 525, 518, 508, 559, 552, 562, 550, 483, 511, 536, 518, 483, 476, 550, 490, 517, 522, 509, 495, 503, 495, 497, 493, 484, 532, 478, 517, 525, 491] payload_rows_per_dst=[5522, 5478, 5492, 5626, 5688, 5608, 5576, 5504] dup_rows_per_dst=[2628, 2646, 2608, 2676, 2738, 2630, 2586, 2530] combine_rows_per_src=[5522, 5478, 5492, 5626, 5688, 5608, 5576, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44368 payload_rows_remote=38768 dup_rows=21168 dup_rows_remote=18370 dedup_ratio=0.322998 router_tokens_per_expert=[504, 508, 526, 511, 521, 512, 504, 497, 498, 472, 503, 526, 525, 509, 494, 548, 522, 498, 498, 466, 521, 480, 496, 503, 560, 544, 531, 503, 502, 526, 520, 513, 513, 503, 499, 475, 517, 530, 513, 519, 485, 459, 583, 539, 503, 528, 542, 490, 491, 512, 461, 502, 498, 511, 493, 541, 547, 513, 517, 523, 555, 503, 553, 509] payload_rows_per_dst=[5514, 5514, 5454, 5600, 5504, 5632, 5496, 5654] dup_rows_per_dst=[2652, 2636, 2514, 2798, 2634, 2626, 2522, 2786] combine_rows_per_src=[5514, 5514, 5454, 5600, 5504, 5632, 5496, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38938 dup_rows=21068 dup_rows_remote=18380 dedup_ratio=0.321472 router_tokens_per_expert=[477, 534, 503, 494, 495, 547, 511, 527, 490, 486, 511, 473, 571, 506, 504, 523, 531, 496, 506, 557, 519, 479, 521, 490, 509, 500, 517, 520, 507, 492, 511, 537, 518, 505, 474, 507, 494, 510, 545, 498, 505, 525, 552, 529, 522, 524, 462, 532, 484, 511, 543, 520, 495, 511, 523, 527, 532, 542, 495, 485, 516, 524, 515, 499] payload_rows_per_dst=[5444, 5404, 5608, 5528, 5540, 5730, 5684, 5530] dup_rows_per_dst=[2502, 2420, 2590, 2658, 2560, 2802, 2848, 2688] combine_rows_per_src=[5444, 5404, 5608, 5528, 5540, 5730, 5684, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38772 dup_rows=21116 dup_rows_remote=18498 dedup_ratio=0.322205 router_tokens_per_expert=[510, 512, 530, 514, 497, 526, 528, 512, 509, 515, 502, 494, 534, 515, 524, 510, 523, 543, 517, 493, 498, 517, 501, 512, 504, 488, 496, 492, 504, 490, 482, 471, 501, 507, 510, 519, 493, 489, 520, 519, 514, 536, 532, 511, 480, 526, 498, 536, 495, 507, 543, 522, 511, 506, 513, 541, 518, 528, 504, 556, 516, 532, 468, 554] payload_rows_per_dst=[5568, 5590, 5526, 5386, 5516, 5648, 5584, 5602] dup_rows_per_dst=[2690, 2616, 2682, 2468, 2600, 2618, 2692, 2750] combine_rows_per_src=[5568, 5590, 5526, 5386, 5516, 5648, 5584, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159907840 dispatch_meta_bytes=229976 combine_payload_bytes=159907840 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160137816 ingress_bytes=235405312 link_bytes=235405312 token_bytes=320045656 weight_movement_bytes=75497472 total_nvlink_bytes=395543128 weight_to_token_ratio=0.235896 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791162429e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159604736 dispatch_meta_bytes=230024 combine_payload_bytes=159604736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159834760 ingress_bytes=184770560 link_bytes=184770560 token_bytes=319439496 weight_movement_bytes=25165824 total_nvlink_bytes=344605320 weight_to_token_ratio=0.078781 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405890471e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160456704 dispatch_meta_bytes=230104 combine_payload_bytes=160456704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160686808 ingress_bytes=185622528 link_bytes=185622528 token_bytes=321143512 weight_movement_bytes=25165824 total_nvlink_bytes=346309336 weight_to_token_ratio=0.078363 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412372963e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159809536 dispatch_meta_bytes=229648 combine_payload_bytes=159809536 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160039184 ingress_bytes=210141184 link_bytes=210141184 token_bytes=319848720 weight_movement_bytes=50331648 total_nvlink_bytes=370180368 weight_to_token_ratio=0.157361 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598931606e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38902 dup_rows=21120 dup_rows_remote=18408 dedup_ratio=0.322266 router_tokens_per_expert=[508, 486, 513, 502, 519, 536, 497, 522, 488, 512, 515, 513, 514, 523, 547, 496, 510, 523, 540, 502, 504, 493, 536, 519, 570, 525, 485, 504, 515, 490, 529, 461, 494, 516, 521, 518, 560, 538, 522, 532, 531, 501, 525, 527, 530, 547, 546, 473, 509, 539, 507, 447, 483, 469, 481, 554, 469, 507, 514, 522, 505, 475, 520, 489] payload_rows_per_dst=[5514, 5556, 5804, 5580, 5672, 5382, 5460, 5448] dup_rows_per_dst=[2712, 2664, 3056, 2588, 2716, 2372, 2518, 2494] combine_rows_per_src=[5514, 5556, 5804, 5580, 5672, 5382, 5460, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158793728 dispatch_meta_bytes=228552 combine_payload_bytes=158793728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159022280 ingress_bytes=183959552 link_bytes=183959552 token_bytes=317816008 weight_movement_bytes=25165824 total_nvlink_bytes=342981832 weight_to_token_ratio=0.079184 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399719637e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159490048 dispatch_meta_bytes=229272 combine_payload_bytes=159490048 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159719320 ingress_bytes=209821696 link_bytes=209821696 token_bytes=319209368 weight_movement_bytes=50331648 total_nvlink_bytes=369541016 weight_to_token_ratio=0.157676 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596500671e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158810112 dispatch_meta_bytes=229080 combine_payload_bytes=158810112 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159039192 ingress_bytes=209141760 link_bytes=209141760 token_bytes=317849304 weight_movement_bytes=50331648 total_nvlink_bytes=368180952 weight_to_token_ratio=0.158351 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591327144e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=229240 combine_payload_bytes=159342592 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159571832 ingress_bytes=209674240 link_bytes=209674240 token_bytes=318914424 weight_movement_bytes=50331648 total_nvlink_bytes=369246072 weight_to_token_ratio=0.157822 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595378701e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38948 dup_rows=21052 dup_rows_remote=18394 dedup_ratio=0.321228 router_tokens_per_expert=[499, 506, 529, 526, 507, 519, 536, 474, 502, 482, 484, 518, 501, 539, 510, 536, 499, 554, 527, 498, 484, 536, 508, 486, 538, 532, 474, 507, 522, 528, 493, 517, 528, 512, 476, 510, 516, 518, 522, 515, 535, 529, 507, 519, 538, 501, 546, 518, 509, 533, 498, 525, 499, 477, 554, 516, 488, 495, 482, 502, 499, 547, 465, 518] payload_rows_per_dst=[5600, 5480, 5534, 5524, 5536, 5660, 5650, 5500] dup_rows_per_dst=[2592, 2664, 2650, 2698, 2658, 2726, 2572, 2492] combine_rows_per_src=[5600, 5480, 5534, 5524, 5536, 5660, 5650, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=39094 dup_rows=21036 dup_rows_remote=18476 dedup_ratio=0.320984 router_tokens_per_expert=[505, 499, 516, 526, 492, 514, 557, 509, 497, 495, 498, 500, 522, 553, 524, 511, 516, 519, 450, 470, 550, 476, 499, 503, 528, 491, 530, 478, 537, 480, 545, 453, 508, 495, 542, 514, 487, 519, 523, 519, 503, 524, 494, 537, 504, 500, 507, 499, 520, 497, 577, 511, 561, 535, 494, 528, 530, 507, 516, 515, 513, 523, 493, 530] payload_rows_per_dst=[5550, 5510, 5406, 5586, 5586, 5638, 5656, 5568] dup_rows_per_dst=[2686, 2690, 2560, 2498, 2628, 2498, 2790, 2686] combine_rows_per_src=[5550, 5510, 5406, 5586, 5586, 5638, 5656, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38852 dup_rows=21108 dup_rows_remote=18460 dedup_ratio=0.322083 router_tokens_per_expert=[520, 513, 518, 519, 488, 528, 480, 524, 525, 526, 542, 473, 499, 482, 545, 520, 525, 502, 507, 501, 518, 540, 488, 493, 482, 511, 473, 497, 529, 483, 540, 508, 478, 501, 498, 504, 519, 510, 529, 536, 548, 502, 497, 540, 522, 524, 517, 480, 498, 503, 494, 500, 512, 523, 501, 532, 535, 564, 484, 513, 542, 532, 512, 519] payload_rows_per_dst=[5542, 5576, 5548, 5522, 5538, 5578, 5472, 5652] dup_rows_per_dst=[2638, 2648, 2600, 2524, 2612, 2682, 2654, 2750] combine_rows_per_src=[5542, 5576, 5548, 5522, 5538, 5578, 5472, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38946 dup_rows=21002 dup_rows_remote=18434 dedup_ratio=0.320465 router_tokens_per_expert=[538, 514, 510, 521, 529, 531, 495, 521, 528, 512, 471, 513, 511, 481, 531, 478, 479, 539, 486, 521, 477, 499, 506, 520, 486, 499, 550, 518, 539, 515, 519, 535, 486, 498, 499, 547, 493, 517, 523, 487, 502, 540, 486, 503, 563, 519, 491, 507, 482, 534, 549, 488, 514, 534, 471, 506, 534, 525, 507, 560, 522, 480, 504, 525] payload_rows_per_dst=[5580, 5522, 5498, 5610, 5500, 5598, 5588, 5638] dup_rows_per_dst=[2738, 2528, 2556, 2712, 2600, 2624, 2568, 2676] combine_rows_per_src=[5580, 5522, 5498, 5610, 5500, 5598, 5588, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44648 payload_rows_remote=38976 dup_rows=20888 dup_rows_remote=18222 dedup_ratio=0.318726 router_tokens_per_expert=[514, 521, 511, 519, 510, 512, 556, 520, 497, 499, 526, 510, 551, 504, 479, 539, 521, 460, 518, 529, 518, 503, 545, 544, 493, 526, 539, 511, 543, 510, 500, 547, 502, 510, 524, 515, 529, 483, 527, 525, 515, 498, 474, 505, 523, 505, 492, 501, 525, 514, 491, 488, 519, 521, 511, 477, 488, 491, 485, 492, 515, 515, 523, 510] payload_rows_per_dst=[5598, 5556, 5608, 5672, 5572, 5536, 5610, 5496] dup_rows_per_dst=[2728, 2654, 2668, 2666, 2658, 2490, 2482, 2542] combine_rows_per_src=[5598, 5556, 5608, 5672, 5572, 5536, 5610, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44386 payload_rows_remote=38764 dup_rows=21150 dup_rows_remote=18308 dedup_ratio=0.322723 router_tokens_per_expert=[537, 503, 501, 493, 494, 532, 510, 489, 531, 505, 542, 497, 543, 549, 540, 532, 505, 507, 508, 526, 500, 520, 495, 478, 499, 508, 510, 529, 476, 510, 464, 519, 473, 520, 486, 520, 530, 502, 521, 543, 502, 522, 538, 504, 488, 514, 498, 502, 516, 476, 512, 507, 492, 553, 517, 525, 481, 515, 498, 529, 524, 539, 534, 535] payload_rows_per_dst=[5178, 5688, 5828, 5400, 5604, 5532, 5534, 5622] dup_rows_per_dst=[2016, 2790, 3174, 2476, 2586, 2604, 2662, 2842] combine_rows_per_src=[5178, 5688, 5828, 5400, 5604, 5532, 5534, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44654 payload_rows_remote=39028 dup_rows=20882 dup_rows_remote=18196 dedup_ratio=0.318634 router_tokens_per_expert=[516, 521, 490, 515, 500, 510, 515, 513, 517, 492, 507, 534, 511, 504, 497, 510, 508, 476, 518, 470, 469, 487, 500, 523, 528, 536, 533, 547, 517, 508, 518, 518, 525, 522, 498, 488, 533, 479, 524, 500, 549, 529, 550, 540, 475, 490, 508, 515, 533, 508, 488, 504, 514, 522, 492, 486, 533, 506, 514, 494, 530, 525, 541, 545] payload_rows_per_dst=[5578, 5562, 5514, 5672, 5574, 5626, 5484, 5644] dup_rows_per_dst=[2582, 2582, 2388, 2738, 2564, 2686, 2610, 2732] combine_rows_per_src=[5578, 5562, 5514, 5672, 5574, 5626, 5484, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159531008 dispatch_meta_bytes=229368 combine_payload_bytes=159531008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159760376 ingress_bytes=184696832 link_bytes=184696832 token_bytes=319291384 weight_movement_bytes=25165824 total_nvlink_bytes=344457208 weight_to_token_ratio=0.078818 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405329486e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159137792 dispatch_meta_bytes=229248 combine_payload_bytes=159137792 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159367040 ingress_bytes=209469440 link_bytes=209469440 token_bytes=318504832 weight_movement_bytes=50331648 total_nvlink_bytes=368836480 weight_to_token_ratio=0.158025 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593820410e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=228792 combine_payload_bytes=159645696 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159874488 ingress_bytes=209977344 link_bytes=209977344 token_bytes=319520184 weight_movement_bytes=50331648 total_nvlink_bytes=369851832 weight_to_token_ratio=0.157523 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597684972e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160129024 dispatch_meta_bytes=230280 combine_payload_bytes=160129024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160359304 ingress_bytes=185294848 link_bytes=185294848 token_bytes=320488328 weight_movement_bytes=25165824 total_nvlink_bytes=345654152 weight_to_token_ratio=0.078523 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409879697e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44328 payload_rows_remote=38508 dup_rows=21208 dup_rows_remote=18098 dedup_ratio=0.323608 router_tokens_per_expert=[515, 515, 528, 502, 529, 530, 523, 548, 542, 543, 485, 494, 499, 505, 539, 566, 502, 506, 488, 499, 485, 494, 544, 484, 514, 494, 502, 511, 472, 512, 533, 549, 547, 500, 521, 501, 522, 515, 518, 527, 491, 486, 494, 526, 488, 505, 513, 535, 495, 517, 523, 498, 539, 549, 494, 519, 496, 533, 516, 532, 479, 501, 478, 458] payload_rows_per_dst=[5820, 5460, 5494, 5614, 5586, 5544, 5554, 5256] dup_rows_per_dst=[3110, 2626, 2510, 2624, 2734, 2514, 2650, 2440] combine_rows_per_src=[5820, 5460, 5494, 5614, 5586, 5544, 5554, 5256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229520 combine_payload_bytes=159522816 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159752336 ingress_bytes=184688640 link_bytes=184688640 token_bytes=319275152 weight_movement_bytes=25165824 total_nvlink_bytes=344440976 weight_to_token_ratio=0.078822 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405267154e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158777344 dispatch_meta_bytes=228288 combine_payload_bytes=158777344 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159005632 ingress_bytes=209108992 link_bytes=209108992 token_bytes=317782976 weight_movement_bytes=50331648 total_nvlink_bytes=368114624 weight_to_token_ratio=0.158384 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591077817e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159858688 dispatch_meta_bytes=228896 combine_payload_bytes=159858688 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160087584 ingress_bytes=185024512 link_bytes=185024512 token_bytes=319946272 weight_movement_bytes=25165824 total_nvlink_bytes=345112096 weight_to_token_ratio=0.078656 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407822752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=157728768 dispatch_meta_bytes=226424 combine_payload_bytes=157728768 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=157955192 ingress_bytes=208060416 link_bytes=208060416 token_bytes=315683960 weight_movement_bytes=50331648 total_nvlink_bytes=366015608 weight_to_token_ratio=0.159437 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.583099365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44404 payload_rows_remote=38874 dup_rows=21132 dup_rows_remote=18600 dedup_ratio=0.322449 router_tokens_per_expert=[528, 491, 491, 528, 512, 501, 540, 541, 526, 544, 524, 504, 550, 519, 510, 482, 526, 496, 507, 484, 514, 459, 515, 511, 493, 519, 481, 529, 518, 536, 524, 505, 510, 488, 482, 524, 513, 515, 500, 499, 513, 523, 512, 513, 491, 509, 509, 516, 508, 518, 509, 511, 496, 567, 518, 532, 506, 499, 495, 509, 509, 517, 514, 535] payload_rows_per_dst=[5602, 5608, 5466, 5572, 5530, 5510, 5574, 5542] dup_rows_per_dst=[2662, 2710, 2558, 2638, 2532, 2662, 2744, 2626] combine_rows_per_src=[5602, 5608, 5466, 5572, 5530, 5510, 5574, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44574 payload_rows_remote=38924 dup_rows=20962 dup_rows_remote=18364 dedup_ratio=0.319855 router_tokens_per_expert=[547, 493, 515, 494, 519, 517, 515, 529, 534, 535, 528, 527, 494, 496, 504, 506, 505, 496, 517, 500, 521, 471, 509, 522, 497, 539, 513, 511, 480, 511, 540, 532, 489, 507, 551, 542, 520, 514, 532, 542, 532, 478, 495, 488, 524, 520, 517, 518, 463, 484, 478, 542, 528, 498, 507, 522, 493, 505, 497, 518, 526, 503, 512, 506] payload_rows_per_dst=[5558, 5650, 5530, 5568, 5696, 5532, 5488, 5552] dup_rows_per_dst=[2700, 2598, 2552, 2678, 2698, 2612, 2556, 2568] combine_rows_per_src=[5558, 5650, 5530, 5568, 5696, 5532, 5488, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44700 payload_rows_remote=39042 dup_rows=20836 dup_rows_remote=18328 dedup_ratio=0.317932 router_tokens_per_expert=[513, 524, 546, 494, 503, 545, 526, 476, 463, 509, 493, 520, 490, 502, 518, 533, 524, 524, 505, 531, 497, 516, 484, 502, 527, 522, 517, 494, 505, 528, 544, 503, 559, 501, 520, 489, 538, 502, 465, 486, 510, 485, 504, 510, 524, 522, 483, 532, 548, 549, 498, 507, 512, 505, 503, 521, 524, 526, 484, 505, 504, 534, 505, 535] payload_rows_per_dst=[5542, 5504, 5658, 5642, 5582, 5572, 5610, 5590] dup_rows_per_dst=[2712, 2552, 2508, 2638, 2538, 2568, 2676, 2644] combine_rows_per_src=[5542, 5504, 5658, 5642, 5582, 5572, 5610, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44578 payload_rows_remote=38886 dup_rows=20958 dup_rows_remote=18256 dedup_ratio=0.319794 router_tokens_per_expert=[501, 487, 528, 517, 517, 483, 504, 510, 525, 555, 503, 513, 492, 529, 516, 493, 505, 525, 478, 513, 514, 533, 493, 530, 532, 517, 522, 505, 498, 524, 553, 546, 527, 519, 499, 485, 523, 515, 511, 518, 515, 484, 490, 508, 514, 470, 524, 524, 490, 544, 501, 525, 498, 461, 519, 490, 486, 559, 550, 503, 539, 500, 525, 491] payload_rows_per_dst=[5504, 5570, 5588, 5692, 5592, 5488, 5522, 5622] dup_rows_per_dst=[2590, 2682, 2594, 2702, 2602, 2570, 2534, 2684] combine_rows_per_src=[5504, 5570, 5588, 5692, 5592, 5488, 5522, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44372 payload_rows_remote=38890 dup_rows=21164 dup_rows_remote=18504 dedup_ratio=0.322937 router_tokens_per_expert=[489, 551, 520, 490, 518, 512, 541, 514, 496, 482, 522, 483, 510, 491, 484, 496, 517, 519, 537, 496, 487, 541, 508, 539, 545, 543, 491, 546, 479, 513, 507, 527, 531, 534, 472, 518, 508, 533, 543, 514, 510, 512, 524, 519, 496, 534, 517, 497, 508, 505, 540, 472, 484, 523, 518, 521, 500, 518, 517, 501, 503, 489, 491, 522] payload_rows_per_dst=[5578, 5450, 5626, 5576, 5620, 5564, 5482, 5476] dup_rows_per_dst=[2692, 2478, 2662, 2726, 2686, 2654, 2660, 2606] combine_rows_per_src=[5578, 5450, 5626, 5576, 5620, 5564, 5482, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38854 dup_rows=21052 dup_rows_remote=18420 dedup_ratio=0.321228 router_tokens_per_expert=[497, 525, 504, 508, 526, 496, 507, 518, 558, 505, 504, 511, 537, 488, 522, 522, 493, 495, 491, 515, 509, 477, 486, 494, 546, 531, 519, 506, 525, 454, 555, 533, 502, 512, 545, 464, 535, 510, 507, 495, 500, 514, 513, 521, 516, 528, 525, 468, 504, 524, 554, 479, 537, 501, 514, 516, 495, 537, 531, 525, 509, 531, 473, 526] payload_rows_per_dst=[5546, 5654, 5438, 5496, 5556, 5552, 5612, 5630] dup_rows_per_dst=[2616, 2640, 2482, 2544, 2584, 2742, 2812, 2632] combine_rows_per_src=[5546, 5654, 5438, 5496, 5556, 5552, 5612, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159227904 dispatch_meta_bytes=229896 combine_payload_bytes=159227904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159457800 ingress_bytes=184393728 link_bytes=184393728 token_bytes=318685704 weight_movement_bytes=25165824 total_nvlink_bytes=343851528 weight_to_token_ratio=0.078968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403023215e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44384 payload_rows_remote=38758 dup_rows=21152 dup_rows_remote=18468 dedup_ratio=0.322754 router_tokens_per_expert=[519, 506, 484, 529, 501, 457, 522, 540, 521, 508, 530, 463, 497, 560, 512, 541, 475, 510, 484, 486, 531, 511, 527, 515, 538, 550, 518, 513, 506, 518, 484, 512, 507, 543, 471, 509, 527, 511, 526, 492, 538, 573, 496, 501, 500, 522, 530, 495, 491, 491, 552, 491, 495, 536, 481, 527, 519, 532, 518, 495, 519, 474, 492, 546] payload_rows_per_dst=[5520, 5636, 5506, 5612, 5460, 5626, 5536, 5488] dup_rows_per_dst=[2596, 2628, 2572, 2666, 2712, 2684, 2592, 2702] combine_rows_per_src=[5520, 5636, 5506, 5612, 5460, 5626, 5536, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=229152 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159661856 ingress_bytes=209764352 link_bytes=209764352 token_bytes=319094560 weight_movement_bytes=50331648 total_nvlink_bytes=369426208 weight_to_token_ratio=0.157733 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596064349e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38992 dup_rows=21000 dup_rows_remote=18426 dedup_ratio=0.320435 router_tokens_per_expert=[505, 476, 463, 511, 504, 450, 561, 537, 510, 507, 521, 519, 544, 509, 507, 521, 545, 515, 531, 518, 491, 497, 521, 509, 494, 550, 511, 530, 516, 514, 505, 546, 503, 535, 528, 499, 472, 492, 533, 494, 515, 506, 510, 509, 512, 542, 485, 521, 526, 481, 503, 491, 532, 523, 512, 501, 522, 521, 493, 521, 491, 556, 464, 537] payload_rows_per_dst=[5544, 5568, 5832, 5360, 5512, 5612, 5566, 5542] dup_rows_per_dst=[2574, 2608, 2964, 2426, 2636, 2588, 2572, 2632] combine_rows_per_src=[5544, 5568, 5832, 5360, 5512, 5612, 5566, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159916032 dispatch_meta_bytes=229480 combine_payload_bytes=159916032 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160145512 ingress_bytes=185081856 link_bytes=185081856 token_bytes=320061544 weight_movement_bytes=25165824 total_nvlink_bytes=345227368 weight_to_token_ratio=0.078628 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408259074e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=229576 combine_payload_bytes=159293440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159523016 ingress_bytes=184459264 link_bytes=184459264 token_bytes=318816456 weight_movement_bytes=25165824 total_nvlink_bytes=343982280 weight_to_token_ratio=0.078935 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403521868e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=228568 combine_payload_bytes=159277056 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=159505624 ingress_bytes=285106176 link_bytes=285106176 token_bytes=318782680 weight_movement_bytes=125829120 total_nvlink_bytes=444611800 weight_to_token_ratio=0.394718 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.169328577e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159145984 dispatch_meta_bytes=229096 combine_payload_bytes=159145984 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159375080 ingress_bytes=209477632 link_bytes=209477632 token_bytes=318521064 weight_movement_bytes=50331648 total_nvlink_bytes=368852712 weight_to_token_ratio=0.158017 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593882741e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158752768 dispatch_meta_bytes=228904 combine_payload_bytes=158752768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158981672 ingress_bytes=183918592 link_bytes=183918592 token_bytes=317734440 weight_movement_bytes=25165824 total_nvlink_bytes=342900264 weight_to_token_ratio=0.079204 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399407979e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159711232 dispatch_meta_bytes=229672 combine_payload_bytes=159711232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159940904 ingress_bytes=184877056 link_bytes=184877056 token_bytes=319652136 weight_movement_bytes=25165824 total_nvlink_bytes=344817960 weight_to_token_ratio=0.078729 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406700783e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44538 payload_rows_remote=39018 dup_rows=20998 dup_rows_remote=18342 dedup_ratio=0.320404 router_tokens_per_expert=[464, 499, 497, 515, 505, 516, 522, 496, 522, 488, 500, 521, 502, 505, 524, 555, 522, 533, 494, 489, 545, 517, 499, 499, 529, 544, 517, 517, 500, 529, 502, 513, 519, 538, 532, 517, 483, 531, 490, 478, 497, 527, 529, 545, 505, 552, 516, 554, 518, 510, 493, 517, 506, 498, 501, 493, 509, 503, 503, 522, 493, 498, 514, 497] payload_rows_per_dst=[5514, 5652, 5594, 5640, 5520, 5612, 5524, 5482] dup_rows_per_dst=[2514, 2582, 2602, 2662, 2656, 2838, 2548, 2596] combine_rows_per_src=[5514, 5652, 5594, 5640, 5520, 5612, 5524, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38902 dup_rows=21124 dup_rows_remote=18574 dedup_ratio=0.322327 router_tokens_per_expert=[510, 510, 520, 501, 514, 497, 501, 489, 490, 478, 527, 483, 544, 515, 511, 482, 545, 494, 533, 539, 502, 543, 530, 503, 514, 535, 517, 511, 533, 520, 486, 531, 500, 475, 527, 501, 507, 507, 526, 487, 491, 533, 510, 515, 562, 527, 536, 497, 482, 520, 521, 524, 492, 531, 516, 511, 505, 530, 506, 536, 489, 530, 508, 458] payload_rows_per_dst=[5526, 5510, 5592, 5598, 5458, 5654, 5546, 5528] dup_rows_per_dst=[2558, 2550, 2786, 2696, 2602, 2688, 2648, 2596] combine_rows_per_src=[5526, 5510, 5592, 5598, 5458, 5654, 5546, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38964 dup_rows=21046 dup_rows_remote=18444 dedup_ratio=0.321136 router_tokens_per_expert=[526, 507, 520, 514, 526, 493, 562, 537, 508, 501, 497, 496, 500, 519, 501, 464, 514, 470, 529, 505, 545, 466, 526, 509, 514, 523, 533, 543, 533, 571, 502, 511, 545, 508, 503, 500, 525, 501, 479, 526, 505, 479, 496, 524, 515, 494, 489, 568, 486, 508, 482, 523, 477, 523, 531, 481, 521, 551, 493, 509, 481, 521, 503, 556] payload_rows_per_dst=[5572, 5458, 5526, 5690, 5596, 5548, 5538, 5562] dup_rows_per_dst=[2798, 2514, 2602, 2770, 2578, 2592, 2484, 2708] combine_rows_per_src=[5572, 5458, 5526, 5690, 5596, 5548, 5538, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38914 dup_rows=21108 dup_rows_remote=18476 dedup_ratio=0.322083 router_tokens_per_expert=[524, 493, 513, 498, 533, 483, 499, 501, 531, 518, 500, 521, 509, 499, 492, 470, 514, 518, 526, 509, 512, 518, 511, 499, 498, 495, 504, 529, 530, 494, 531, 511, 488, 559, 496, 527, 542, 524, 530, 511, 508, 535, 532, 505, 515, 507, 519, 490, 520, 533, 503, 522, 515, 468, 484, 528, 516, 539, 516, 506, 504, 517, 525, 501] payload_rows_per_dst=[5478, 5482, 5586, 5600, 5600, 5584, 5514, 5584] dup_rows_per_dst=[2610, 2598, 2628, 2584, 2754, 2638, 2632, 2664] combine_rows_per_src=[5478, 5482, 5586, 5600, 5600, 5584, 5514, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=38888 dup_rows=20976 dup_rows_remote=18288 dedup_ratio=0.320068 router_tokens_per_expert=[517, 515, 496, 509, 529, 550, 527, 507, 479, 545, 489, 519, 509, 525, 500, 501, 510, 530, 506, 510, 481, 508, 521, 500, 546, 505, 517, 535, 505, 526, 518, 528, 524, 527, 524, 510, 479, 491, 512, 513, 489, 518, 494, 516, 468, 513, 536, 517, 520, 549, 520, 496, 513, 515, 471, 547, 508, 501, 527, 523, 483, 501, 499, 501] payload_rows_per_dst=[5668, 5588, 5552, 5672, 5556, 5446, 5546, 5532] dup_rows_per_dst=[2632, 2546, 2580, 2688, 2604, 2656, 2716, 2554] combine_rows_per_src=[5668, 5588, 5552, 5672, 5556, 5446, 5546, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44368 payload_rows_remote=38596 dup_rows=21168 dup_rows_remote=18290 dedup_ratio=0.322998 router_tokens_per_expert=[546, 472, 508, 542, 528, 502, 513, 505, 499, 503, 502, 527, 534, 539, 519, 516, 547, 509, 508, 516, 495, 498, 524, 538, 510, 525, 472, 527, 497, 508, 504, 478, 510, 541, 533, 491, 485, 525, 495, 497, 520, 512, 521, 497, 513, 531, 528, 500, 495, 487, 538, 536, 507, 503, 516, 478, 511, 498, 538, 501, 490, 542, 519, 499] payload_rows_per_dst=[5466, 5618, 5594, 5330, 5496, 5338, 5754, 5772] dup_rows_per_dst=[2552, 2660, 2724, 2286, 2700, 2372, 2996, 2878] combine_rows_per_src=[5466, 5618, 5594, 5330, 5496, 5338, 5754, 5772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=229440 combine_payload_bytes=159817728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160047168 ingress_bytes=184983552 link_bytes=184983552 token_bytes=319864896 weight_movement_bytes=25165824 total_nvlink_bytes=345030720 weight_to_token_ratio=0.078676 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407511094e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=38964 dup_rows=20976 dup_rows_remote=18366 dedup_ratio=0.320068 router_tokens_per_expert=[507, 562, 519, 555, 538, 494, 513, 526, 516, 535, 494, 485, 521, 462, 527, 516, 510, 504, 505, 504, 515, 525, 548, 536, 522, 497, 517, 493, 489, 537, 503, 532, 550, 535, 510, 490, 554, 520, 502, 511, 503, 534, 529, 523, 512, 496, 490, 516, 506, 528, 494, 495, 517, 483, 483, 487, 491, 488, 498, 522, 472, 478, 512, 532] payload_rows_per_dst=[5598, 5574, 5610, 5554, 5656, 5596, 5504, 5468] dup_rows_per_dst=[2830, 2538, 2684, 2626, 2688, 2610, 2482, 2518] combine_rows_per_src=[5598, 5574, 5610, 5554, 5656, 5596, 5504, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=229904 combine_payload_bytes=159342592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159572496 ingress_bytes=184508416 link_bytes=184508416 token_bytes=318915088 weight_movement_bytes=25165824 total_nvlink_bytes=344080912 weight_to_token_ratio=0.078911 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403895858e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44408 payload_rows_remote=38810 dup_rows=21128 dup_rows_remote=18506 dedup_ratio=0.322388 router_tokens_per_expert=[526, 511, 547, 530, 485, 475, 496, 540, 486, 520, 551, 526, 524, 541, 497, 502, 533, 506, 495, 510, 516, 534, 523, 507, 543, 506, 497, 504, 499, 494, 528, 499, 495, 509, 512, 473, 474, 520, 531, 522, 540, 502, 543, 509, 476, 513, 560, 539, 515, 492, 512, 512, 475, 543, 504, 499, 549, 497, 499, 503, 498, 501, 509, 491] payload_rows_per_dst=[5598, 5818, 5392, 5522, 5490, 5604, 5512, 5472] dup_rows_per_dst=[2622, 2848, 2484, 2618, 2582, 2760, 2592, 2622] combine_rows_per_src=[5598, 5818, 5392, 5522, 5490, 5604, 5512, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229632 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159826176 ingress_bytes=209928192 link_bytes=209928192 token_bytes=319422720 weight_movement_bytes=50331648 total_nvlink_bytes=369754368 weight_to_token_ratio=0.157571 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597310982e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159391744 dispatch_meta_bytes=229560 combine_payload_bytes=159391744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159621304 ingress_bytes=184557568 link_bytes=184557568 token_bytes=319013048 weight_movement_bytes=25165824 total_nvlink_bytes=344178872 weight_to_token_ratio=0.078887 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404269848e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=228704 combine_payload_bytes=159285248 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159513952 ingress_bytes=209616896 link_bytes=209616896 token_bytes=318799200 weight_movement_bytes=50331648 total_nvlink_bytes=369130848 weight_to_token_ratio=0.157879 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594942380e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158089216 dispatch_meta_bytes=227544 combine_payload_bytes=158089216 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158316760 ingress_bytes=183255040 link_bytes=183255040 token_bytes=316405976 weight_movement_bytes=25165824 total_nvlink_bytes=341571800 weight_to_token_ratio=0.079537 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.394359115e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229320 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159825864 ingress_bytes=209928192 link_bytes=209928192 token_bytes=319422408 weight_movement_bytes=50331648 total_nvlink_bytes=369754056 weight_to_token_ratio=0.157571 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597310982e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158965760 dispatch_meta_bytes=229264 combine_payload_bytes=158965760 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159195024 ingress_bytes=259629056 link_bytes=259629056 token_bytes=318160784 weight_movement_bytes=100663296 total_nvlink_bytes=418824080 weight_to_token_ratio=0.316391 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975477131e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44336 payload_rows_remote=38744 dup_rows=21200 dup_rows_remote=18568 dedup_ratio=0.323486 router_tokens_per_expert=[498, 512, 574, 505, 503, 497, 494, 528, 524, 520, 494, 531, 514, 515, 503, 522, 506, 494, 552, 493, 485, 494, 496, 492, 531, 534, 513, 512, 484, 520, 521, 524, 530, 513, 513, 507, 545, 502, 519, 483, 463, 547, 539, 519, 500, 527, 526, 535, 519, 508, 491, 533, 464, 462, 512, 486, 523, 496, 523, 529, 512, 525, 499, 533] payload_rows_per_dst=[5538, 5596, 5462, 5560, 5592, 5574, 5424, 5590] dup_rows_per_dst=[2684, 2650, 2562, 2718, 2632, 2738, 2526, 2690] combine_rows_per_src=[5538, 5596, 5462, 5560, 5592, 5574, 5424, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44408 payload_rows_remote=38838 dup_rows=21128 dup_rows_remote=18468 dedup_ratio=0.322388 router_tokens_per_expert=[487, 513, 517, 540, 504, 527, 500, 541, 528, 513, 563, 517, 493, 476, 500, 525, 520, 512, 518, 508, 467, 518, 515, 523, 503, 505, 537, 531, 522, 526, 472, 523, 484, 523, 558, 519, 450, 550, 541, 516, 503, 503, 494, 515, 518, 516, 508, 471, 542, 510, 519, 540, 526, 515, 509, 500, 504, 508, 530, 509, 420, 512, 517, 494] payload_rows_per_dst=[5622, 5570, 5562, 5542, 5640, 5470, 5594, 5408] dup_rows_per_dst=[2636, 2660, 2600, 2696, 2642, 2586, 2728, 2580] combine_rows_per_src=[5622, 5570, 5562, 5542, 5640, 5470, 5594, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38896 dup_rows=21046 dup_rows_remote=18412 dedup_ratio=0.321136 router_tokens_per_expert=[513, 546, 538, 524, 477, 518, 542, 513, 457, 481, 515, 493, 485, 494, 519, 527, 511, 523, 528, 479, 507, 523, 517, 526, 500, 473, 554, 537, 520, 520, 493, 501, 500, 539, 501, 505, 492, 504, 506, 502, 514, 520, 535, 485, 555, 496, 522, 488, 491, 528, 515, 534, 515, 499, 486, 537, 529, 496, 508, 504, 535, 522, 534, 517] payload_rows_per_dst=[5624, 5438, 5594, 5540, 5488, 5582, 5594, 5630] dup_rows_per_dst=[2718, 2504, 2634, 2656, 2610, 2648, 2616, 2660] combine_rows_per_src=[5624, 5438, 5594, 5540, 5488, 5582, 5594, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44646 payload_rows_remote=39086 dup_rows=20890 dup_rows_remote=18300 dedup_ratio=0.318756 router_tokens_per_expert=[547, 525, 511, 488, 552, 502, 458, 489, 505, 527, 503, 515, 487, 480, 491, 518, 522, 517, 539, 512, 515, 502, 499, 492, 521, 520, 535, 529, 496, 521, 476, 491, 501, 526, 536, 517, 541, 525, 538, 519, 493, 538, 504, 521, 527, 495, 520, 526, 517, 536, 512, 479, 511, 516, 505, 499, 494, 508, 525, 487, 524, 521, 518, 504] payload_rows_per_dst=[5544, 5484, 5604, 5602, 5654, 5626, 5560, 5572] dup_rows_per_dst=[2600, 2568, 2592, 2576, 2752, 2622, 2590, 2590] combine_rows_per_src=[5544, 5484, 5604, 5602, 5654, 5626, 5560, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44618 payload_rows_remote=38944 dup_rows=20918 dup_rows_remote=18190 dedup_ratio=0.319183 router_tokens_per_expert=[552, 534, 489, 501, 517, 515, 488, 517, 529, 551, 484, 496, 483, 506, 532, 509, 514, 506, 514, 538, 505, 503, 495, 486, 490, 524, 557, 548, 524, 519, 501, 538, 519, 545, 505, 495, 476, 467, 527, 497, 538, 478, 545, 503, 514, 500, 521, 518, 540, 529, 524, 547, 517, 490, 522, 496, 474, 545, 499, 492, 505, 506, 462, 507] payload_rows_per_dst=[5620, 5580, 5518, 5674, 5516, 5592, 5690, 5428] dup_rows_per_dst=[2606, 2600, 2604, 2728, 2546, 2642, 2640, 2552] combine_rows_per_src=[5620, 5580, 5518, 5674, 5516, 5592, 5690, 5428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158695424 dispatch_meta_bytes=229248 combine_payload_bytes=158695424 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158924672 ingress_bytes=183861248 link_bytes=183861248 token_bytes=317620096 weight_movement_bytes=25165824 total_nvlink_bytes=342785920 weight_to_token_ratio=0.079232 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398971657e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44410 payload_rows_remote=38866 dup_rows=21126 dup_rows_remote=18544 dedup_ratio=0.322357 router_tokens_per_expert=[470, 531, 516, 501, 498, 520, 509, 529, 540, 487, 508, 533, 512, 522, 516, 503, 508, 493, 518, 534, 551, 533, 513, 486, 541, 511, 510, 501, 536, 495, 494, 520, 538, 494, 437, 504, 482, 506, 531, 519, 523, 508, 508, 491, 499, 510, 505, 519, 530, 504, 549, 522, 518, 519, 502, 504, 480, 547, 482, 531, 521, 516, 537, 493] payload_rows_per_dst=[5536, 5552, 5590, 5526, 5532, 5544, 5576, 5554] dup_rows_per_dst=[2612, 2690, 2682, 2690, 2490, 2582, 2720, 2660] combine_rows_per_src=[5536, 5552, 5590, 5526, 5532, 5544, 5576, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=38880 dup_rows=20916 dup_rows_remote=18044 dedup_ratio=0.319153 router_tokens_per_expert=[510, 527, 501, 506, 539, 490, 543, 483, 545, 500, 512, 517, 497, 525, 543, 523, 481, 480, 541, 527, 500, 496, 515, 536, 526, 507, 510, 488, 549, 535, 484, 509, 520, 533, 528, 511, 471, 484, 547, 461, 486, 519, 519, 510, 503, 508, 521, 537, 521, 536, 529, 512, 522, 535, 494, 502, 499, 501, 482, 530, 483, 524, 519, 476] payload_rows_per_dst=[5480, 5690, 5582, 5410, 5546, 5544, 5628, 5740] dup_rows_per_dst=[2474, 2684, 2570, 2416, 2564, 2654, 2682, 2872] combine_rows_per_src=[5480, 5690, 5582, 5410, 5546, 5544, 5628, 5740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159080448 dispatch_meta_bytes=229224 combine_payload_bytes=159080448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159309672 ingress_bytes=184246272 link_bytes=184246272 token_bytes=318390120 weight_movement_bytes=25165824 total_nvlink_bytes=343555944 weight_to_token_ratio=0.079041 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401901245e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159318016 dispatch_meta_bytes=229232 combine_payload_bytes=159318016 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159547248 ingress_bytes=184483840 link_bytes=184483840 token_bytes=318865264 weight_movement_bytes=25165824 total_nvlink_bytes=344031088 weight_to_token_ratio=0.078923 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403708863e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=160096256 dispatch_meta_bytes=229544 combine_payload_bytes=160096256 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160325800 ingress_bytes=235593728 link_bytes=235593728 token_bytes=320422056 weight_movement_bytes=75497472 total_nvlink_bytes=395919528 weight_to_token_ratio=0.235619 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792596057e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159514624 dispatch_meta_bytes=228536 combine_payload_bytes=159514624 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159743160 ingress_bytes=209846272 link_bytes=209846272 token_bytes=319257784 weight_movement_bytes=50331648 total_nvlink_bytes=369589432 weight_to_token_ratio=0.157652 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596687666e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159195136 dispatch_meta_bytes=229640 combine_payload_bytes=159195136 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159424776 ingress_bytes=209526784 link_bytes=209526784 token_bytes=318619912 weight_movement_bytes=50331648 total_nvlink_bytes=368951560 weight_to_token_ratio=0.157968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594256731e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44714 payload_rows_remote=39112 dup_rows=20822 dup_rows_remote=18182 dedup_ratio=0.317719 router_tokens_per_expert=[491, 527, 543, 493, 529, 504, 527, 498, 515, 559, 540, 502, 503, 496, 507, 513, 561, 513, 486, 516, 480, 521, 507, 516, 517, 505, 501, 544, 484, 492, 515, 543, 474, 523, 528, 491, 513, 528, 526, 538, 527, 478, 520, 528, 506, 547, 501, 504, 487, 515, 503, 514, 512, 488, 541, 523, 453, 501, 510, 507, 510, 514, 484, 526] payload_rows_per_dst=[5602, 5588, 5672, 5618, 5646, 5584, 5460, 5544] dup_rows_per_dst=[2640, 2682, 2708, 2584, 2686, 2548, 2508, 2466] combine_rows_per_src=[5602, 5588, 5672, 5618, 5646, 5584, 5460, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159252480 dispatch_meta_bytes=227696 combine_payload_bytes=159252480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159480176 ingress_bytes=184418304 link_bytes=184418304 token_bytes=318732656 weight_movement_bytes=25165824 total_nvlink_bytes=343898480 weight_to_token_ratio=0.078956 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403210210e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44400 payload_rows_remote=38820 dup_rows=21136 dup_rows_remote=18446 dedup_ratio=0.322510 router_tokens_per_expert=[524, 498, 509, 554, 487, 506, 492, 544, 511, 514, 503, 494, 494, 488, 481, 498, 492, 528, 540, 513, 478, 526, 486, 493, 546, 480, 521, 561, 467, 513, 537, 532, 519, 497, 502, 522, 520, 525, 517, 533, 554, 499, 517, 532, 540, 501, 551, 493, 516, 479, 532, 497, 515, 526, 500, 500, 526, 536, 520, 518, 518, 498, 465, 490] payload_rows_per_dst=[5624, 5422, 5510, 5618, 5580, 5594, 5502, 5550] dup_rows_per_dst=[2604, 2544, 2602, 2696, 2690, 2780, 2628, 2592] combine_rows_per_src=[5624, 5422, 5510, 5618, 5580, 5594, 5502, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=160202752 dispatch_meta_bytes=229176 combine_payload_bytes=160202752 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160431928 ingress_bytes=235700224 link_bytes=235700224 token_bytes=320634680 weight_movement_bytes=75497472 total_nvlink_bytes=396132152 weight_to_token_ratio=0.235463 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.793406368e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44588 payload_rows_remote=39034 dup_rows=20948 dup_rows_remote=18414 dedup_ratio=0.319641 router_tokens_per_expert=[514, 523, 471, 522, 487, 524, 522, 491, 475, 504, 519, 510, 504, 496, 540, 496, 499, 513, 500, 516, 478, 478, 532, 491, 516, 544, 509, 520, 492, 509, 525, 519, 503, 520, 500, 537, 566, 503, 522, 511, 509, 513, 516, 466, 511, 526, 557, 519, 535, 487, 517, 504, 487, 494, 538, 563, 518, 470, 515, 533, 544, 507, 477, 561] payload_rows_per_dst=[5534, 5554, 5466, 5638, 5626, 5580, 5598, 5592] dup_rows_per_dst=[2574, 2534, 2548, 2630, 2698, 2654, 2652, 2658] combine_rows_per_src=[5534, 5554, 5466, 5638, 5626, 5580, 5598, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=39010 dup_rows=20968 dup_rows_remote=18398 dedup_ratio=0.319946 router_tokens_per_expert=[520, 515, 521, 535, 541, 512, 490, 506, 499, 544, 498, 484, 510, 548, 559, 486, 514, 474, 514, 525, 508, 511, 514, 504, 496, 538, 530, 560, 506, 536, 499, 525, 505, 492, 519, 503, 504, 521, 501, 501, 512, 507, 548, 449, 532, 498, 502, 537, 501, 497, 523, 485, 494, 527, 519, 529, 467, 510, 517, 502, 494, 519, 517, 514] payload_rows_per_dst=[5636, 5564, 5558, 5650, 5576, 5574, 5560, 5450] dup_rows_per_dst=[2644, 2692, 2570, 2730, 2516, 2596, 2590, 2630] combine_rows_per_src=[5636, 5564, 5558, 5650, 5576, 5574, 5560, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44766 payload_rows_remote=39188 dup_rows=20770 dup_rows_remote=18188 dedup_ratio=0.316925 router_tokens_per_expert=[501, 535, 533, 523, 554, 550, 519, 499, 501, 506, 520, 545, 521, 522, 494, 518, 538, 507, 522, 489, 493, 546, 504, 523, 519, 530, 507, 483, 506, 514, 500, 502, 506, 473, 512, 531, 512, 534, 528, 517, 503, 473, 479, 456, 486, 517, 511, 499, 533, 536, 485, 478, 518, 535, 497, 498, 507, 512, 534, 525, 486, 500, 549, 514] payload_rows_per_dst=[5694, 5634, 5620, 5586, 5578, 5436, 5578, 5640] dup_rows_per_dst=[2734, 2620, 2624, 2536, 2648, 2412, 2582, 2614] combine_rows_per_src=[5694, 5634, 5620, 5586, 5578, 5436, 5578, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44402 payload_rows_remote=38812 dup_rows=21134 dup_rows_remote=18500 dedup_ratio=0.322479 router_tokens_per_expert=[506, 518, 508, 500, 530, 485, 511, 522, 488, 507, 515, 527, 499, 532, 506, 518, 513, 516, 500, 490, 507, 562, 491, 532, 520, 533, 525, 507, 520, 500, 488, 519, 507, 524, 535, 529, 493, 541, 500, 542, 519, 529, 510, 490, 529, 540, 498, 495, 522, 507, 456, 546, 522, 481, 500, 514, 526, 494, 479, 503, 511, 521, 504, 506] payload_rows_per_dst=[5544, 5546, 5534, 5590, 5578, 5582, 5504, 5524] dup_rows_per_dst=[2616, 2638, 2688, 2634, 2764, 2638, 2592, 2564] combine_rows_per_src=[5544, 5546, 5534, 5590, 5578, 5582, 5504, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=229064 combine_payload_bytes=159006720 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159235784 ingress_bytes=209338368 link_bytes=209338368 token_bytes=318242504 weight_movement_bytes=50331648 total_nvlink_bytes=368574152 weight_to_token_ratio=0.158155 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592823103e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44326 payload_rows_remote=38680 dup_rows=21210 dup_rows_remote=18374 dedup_ratio=0.323639 router_tokens_per_expert=[497, 531, 520, 523, 524, 493, 520, 547, 530, 531, 508, 547, 497, 499, 522, 501, 510, 507, 523, 529, 499, 509, 526, 497, 522, 490, 543, 481, 533, 505, 516, 483, 505, 495, 501, 502, 510, 543, 503, 550, 543, 493, 519, 503, 510, 509, 485, 505, 516, 505, 523, 485, 546, 531, 532, 469, 510, 510, 515, 489, 529, 488, 489, 492] payload_rows_per_dst=[5344, 5666, 5572, 5436, 5444, 5580, 5638, 5646] dup_rows_per_dst=[2344, 2788, 2628, 2540, 2580, 2646, 2848, 2836] combine_rows_per_src=[5344, 5666, 5572, 5436, 5444, 5580, 5638, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44570 payload_rows_remote=38912 dup_rows=20966 dup_rows_remote=18286 dedup_ratio=0.319916 router_tokens_per_expert=[485, 547, 516, 556, 517, 523, 536, 532, 489, 536, 499, 491, 558, 494, 480, 527, 525, 532, 488, 478, 547, 497, 513, 520, 503, 506, 514, 544, 522, 513, 484, 535, 488, 511, 530, 496, 491, 528, 476, 492, 503, 541, 520, 522, 540, 510, 505, 528, 495, 512, 526, 505, 505, 513, 464, 511, 480, 507, 541, 467, 500, 526, 511, 517] payload_rows_per_dst=[5630, 5550, 5580, 5622, 5496, 5658, 5492, 5542] dup_rows_per_dst=[2794, 2598, 2620, 2620, 2528, 2680, 2570, 2556] combine_rows_per_src=[5630, 5550, 5580, 5622, 5496, 5658, 5492, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=229792 combine_payload_bytes=159883264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160113056 ingress_bytes=185049088 link_bytes=185049088 token_bytes=319996320 weight_movement_bytes=25165824 total_nvlink_bytes=345162144 weight_to_token_ratio=0.078644 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408009747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=229632 combine_payload_bytes=159784960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160014592 ingress_bytes=184950784 link_bytes=184950784 token_bytes=319799552 weight_movement_bytes=25165824 total_nvlink_bytes=344965376 weight_to_token_ratio=0.078692 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407261767e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=160514048 dispatch_meta_bytes=229504 combine_payload_bytes=160514048 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160743552 ingress_bytes=185679872 link_bytes=185679872 token_bytes=321257600 weight_movement_bytes=25165824 total_nvlink_bytes=346423424 weight_to_token_ratio=0.078335 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412809285e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158973952 dispatch_meta_bytes=229248 combine_payload_bytes=158973952 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159203200 ingress_bytes=259637248 link_bytes=259637248 token_bytes=318177152 weight_movement_bytes=100663296 total_nvlink_bytes=418840448 weight_to_token_ratio=0.316375 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975539463e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158433280 dispatch_meta_bytes=228216 combine_payload_bytes=158433280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158661496 ingress_bytes=183599104 link_bytes=183599104 token_bytes=317094776 weight_movement_bytes=25165824 total_nvlink_bytes=342260600 weight_to_token_ratio=0.079364 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396977044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44318 payload_rows_remote=38772 dup_rows=21218 dup_rows_remote=18544 dedup_ratio=0.323761 router_tokens_per_expert=[541, 496, 456, 502, 519, 544, 503, 548, 516, 516, 491, 545, 514, 496, 512, 503, 545, 527, 491, 505, 487, 514, 517, 461, 526, 514, 523, 567, 529, 466, 534, 508, 499, 525, 488, 493, 511, 493, 573, 535, 498, 515, 507, 516, 486, 514, 494, 533, 496, 512, 518, 524, 537, 506, 493, 515, 497, 495, 527, 520, 506, 518, 481, 527] payload_rows_per_dst=[5546, 5526, 5636, 5462, 5554, 5516, 5598, 5480] dup_rows_per_dst=[2674, 2660, 2780, 2548, 2680, 2610, 2604, 2662] combine_rows_per_src=[5546, 5526, 5636, 5462, 5554, 5516, 5598, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159383552 dispatch_meta_bytes=228792 combine_payload_bytes=159383552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159612344 ingress_bytes=184549376 link_bytes=184549376 token_bytes=318995896 weight_movement_bytes=25165824 total_nvlink_bytes=344161720 weight_to_token_ratio=0.078891 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404207516e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44378 payload_rows_remote=38722 dup_rows=21158 dup_rows_remote=18510 dedup_ratio=0.322845 router_tokens_per_expert=[514, 512, 516, 516, 534, 491, 511, 517, 535, 487, 514, 470, 518, 515, 513, 480, 500, 470, 508, 516, 488, 517, 518, 567, 566, 573, 518, 522, 514, 496, 476, 509, 519, 507, 537, 508, 532, 511, 536, 502, 482, 523, 490, 511, 518, 511, 515, 507, 527, 488, 526, 526, 492, 474, 507, 554, 483, 538, 528, 523, 525, 493, 499, 475] payload_rows_per_dst=[5518, 5532, 5530, 5600, 5656, 5536, 5484, 5522] dup_rows_per_dst=[2704, 2532, 2638, 2748, 2648, 2578, 2704, 2606] combine_rows_per_src=[5518, 5532, 5530, 5600, 5656, 5536, 5484, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=39018 dup_rows=20916 dup_rows_remote=18262 dedup_ratio=0.319153 router_tokens_per_expert=[480, 507, 498, 519, 502, 525, 510, 489, 495, 505, 515, 501, 513, 545, 531, 523, 489, 518, 535, 549, 571, 508, 523, 507, 513, 516, 502, 538, 526, 500, 521, 500, 496, 505, 534, 498, 499, 467, 531, 529, 504, 497, 521, 541, 478, 518, 513, 504, 511, 532, 497, 528, 505, 452, 526, 495, 519, 499, 503, 530, 475, 506, 550, 531] payload_rows_per_dst=[5518, 5602, 5630, 5574, 5590, 5570, 5546, 5590] dup_rows_per_dst=[2542, 2654, 2770, 2658, 2528, 2582, 2546, 2636] combine_rows_per_src=[5518, 5602, 5630, 5574, 5590, 5570, 5546, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158810112 dispatch_meta_bytes=229264 combine_payload_bytes=158810112 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159039376 ingress_bytes=234307584 link_bytes=234307584 token_bytes=317849488 weight_movement_bytes=75497472 total_nvlink_bytes=393346960 weight_to_token_ratio=0.237526 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782809987e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158605312 dispatch_meta_bytes=228928 combine_payload_bytes=158605312 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158834240 ingress_bytes=208936960 link_bytes=208936960 token_bytes=317439552 weight_movement_bytes=50331648 total_nvlink_bytes=367771200 weight_to_token_ratio=0.158555 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589768852e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=38960 dup_rows=20988 dup_rows_remote=18372 dedup_ratio=0.320251 router_tokens_per_expert=[504, 515, 533, 457, 489, 495, 525, 505, 541, 515, 513, 478, 521, 515, 492, 500, 532, 475, 500, 508, 509, 492, 512, 517, 524, 547, 492, 502, 511, 521, 520, 534, 513, 492, 509, 532, 525, 495, 506, 509, 527, 512, 524, 542, 537, 534, 511, 554, 497, 468, 532, 556, 500, 487, 506, 556, 511, 527, 501, 539, 470, 519, 493, 490] payload_rows_per_dst=[5468, 5512, 5548, 5594, 5596, 5726, 5588, 5516] dup_rows_per_dst=[2578, 2638, 2542, 2708, 2566, 2756, 2616, 2584] combine_rows_per_src=[5468, 5512, 5548, 5594, 5596, 5726, 5588, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38794 dup_rows=21092 dup_rows_remote=18408 dedup_ratio=0.321838 router_tokens_per_expert=[525, 501, 514, 531, 474, 500, 476, 517, 515, 546, 517, 510, 474, 493, 491, 486, 506, 460, 512, 514, 522, 523, 482, 473, 553, 516, 516, 505, 516, 557, 490, 514, 507, 535, 551, 511, 534, 554, 494, 491, 516, 511, 546, 498, 525, 519, 540, 551, 533, 515, 503, 492, 467, 515, 491, 523, 494, 480, 537, 556, 495, 516, 516, 523] payload_rows_per_dst=[5502, 5460, 5520, 5650, 5628, 5606, 5518, 5560] dup_rows_per_dst=[2574, 2604, 2464, 2684, 2726, 2806, 2560, 2674] combine_rows_per_src=[5502, 5460, 5520, 5650, 5628, 5606, 5518, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44456 payload_rows_remote=39020 dup_rows=21080 dup_rows_remote=18436 dedup_ratio=0.321655 router_tokens_per_expert=[490, 497, 521, 509, 514, 497, 525, 499, 487, 526, 485, 547, 518, 536, 522, 520, 503, 511, 502, 481, 538, 492, 517, 496, 524, 530, 528, 501, 532, 518, 488, 510, 551, 508, 520, 526, 479, 498, 530, 504, 494, 501, 551, 518, 510, 471, 506, 543, 509, 490, 536, 505, 505, 554, 516, 509, 493, 535, 493, 532, 521, 519, 473, 504] payload_rows_per_dst=[5518, 5642, 5436, 5650, 5528, 5556, 5486, 5640] dup_rows_per_dst=[2586, 2640, 2644, 2612, 2704, 2632, 2762, 2500] combine_rows_per_src=[5518, 5642, 5436, 5650, 5528, 5556, 5486, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44596 payload_rows_remote=39010 dup_rows=20940 dup_rows_remote=18312 dedup_ratio=0.319519 router_tokens_per_expert=[507, 533, 519, 512, 521, 541, 519, 523, 484, 495, 528, 535, 508, 488, 500, 533, 538, 523, 525, 552, 465, 514, 478, 489, 503, 511, 492, 485, 499, 474, 538, 517, 474, 533, 557, 502, 521, 521, 560, 539, 468, 480, 531, 508, 480, 506, 511, 564, 553, 451, 538, 500, 472, 558, 503, 484, 490, 533, 533, 496, 490, 537, 488, 538] payload_rows_per_dst=[5672, 5532, 5600, 5534, 5414, 5528, 5730, 5586] dup_rows_per_dst=[2678, 2606, 2568, 2504, 2558, 2568, 2830, 2628] combine_rows_per_src=[5672, 5532, 5600, 5534, 5414, 5528, 5730, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=229120 combine_payload_bytes=159817728 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160046848 ingress_bytes=184983552 link_bytes=184983552 token_bytes=319864576 weight_movement_bytes=25165824 total_nvlink_bytes=345030400 weight_to_token_ratio=0.078676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407511094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44612 payload_rows_remote=39144 dup_rows=20924 dup_rows_remote=18430 dedup_ratio=0.319275 router_tokens_per_expert=[516, 504, 480, 496, 499, 539, 528, 515, 548, 509, 522, 518, 513, 528, 513, 541, 530, 526, 513, 525, 497, 543, 511, 495, 493, 491, 517, 493, 509, 559, 517, 506, 511, 474, 500, 495, 508, 516, 535, 530, 484, 509, 532, 492, 500, 463, 477, 524, 525, 493, 475, 547, 515, 508, 505, 475, 548, 498, 525, 520, 508, 530, 518, 534] payload_rows_per_dst=[5568, 5666, 5582, 5592, 5580, 5468, 5466, 5690] dup_rows_per_dst=[2586, 2718, 2698, 2578, 2558, 2494, 2620, 2672] combine_rows_per_src=[5568, 5666, 5582, 5592, 5580, 5468, 5466, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159580160 dispatch_meta_bytes=229328 combine_payload_bytes=159580160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159809488 ingress_bytes=184745984 link_bytes=184745984 token_bytes=319389648 weight_movement_bytes=25165824 total_nvlink_bytes=344555472 weight_to_token_ratio=0.078793 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405703476e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158900224 dispatch_meta_bytes=228808 combine_payload_bytes=158900224 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159129032 ingress_bytes=234397696 link_bytes=234397696 token_bytes=318029256 weight_movement_bytes=75497472 total_nvlink_bytes=393526728 weight_to_token_ratio=0.237392 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783495635e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159825920 dispatch_meta_bytes=229824 combine_payload_bytes=159825920 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160055744 ingress_bytes=184991744 link_bytes=184991744 token_bytes=319881664 weight_movement_bytes=25165824 total_nvlink_bytes=345047488 weight_to_token_ratio=0.078672 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407573426e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=229288 combine_payload_bytes=159784960 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=160014248 ingress_bytes=260448256 link_bytes=260448256 token_bytes=319799208 weight_movement_bytes=100663296 total_nvlink_bytes=420462504 weight_to_token_ratio=0.314770 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.981710297e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=38804 dup_rows=20990 dup_rows_remote=18208 dedup_ratio=0.320282 router_tokens_per_expert=[511, 513, 536, 546, 535, 525, 528, 541, 521, 497, 547, 503, 514, 498, 479, 540, 540, 460, 507, 508, 522, 555, 475, 517, 486, 503, 501, 482, 508, 521, 513, 529, 526, 497, 510, 506, 506, 501, 527, 512, 486, 489, 522, 508, 540, 535, 477, 503, 460, 504, 515, 484, 538, 526, 493, 520, 565, 537, 489, 517, 498, 520, 489, 507] payload_rows_per_dst=[5742, 5564, 5646, 5450, 5630, 5516, 5514, 5484] dup_rows_per_dst=[2782, 2634, 2720, 2400, 2626, 2614, 2566, 2648] combine_rows_per_src=[5742, 5564, 5646, 5450, 5630, 5516, 5514, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160333824 dispatch_meta_bytes=230296 combine_payload_bytes=160333824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160564120 ingress_bytes=185499648 link_bytes=185499648 token_bytes=320897944 weight_movement_bytes=25165824 total_nvlink_bytes=346063768 weight_to_token_ratio=0.078423 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411437988e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44672 payload_rows_remote=39112 dup_rows=20864 dup_rows_remote=18280 dedup_ratio=0.318359 router_tokens_per_expert=[525, 519, 515, 494, 507, 574, 477, 538, 471, 531, 486, 495, 463, 520, 506, 549, 473, 505, 528, 499, 530, 473, 485, 504, 528, 479, 516, 491, 533, 529, 502, 570, 493, 524, 525, 510, 489, 488, 530, 513, 501, 509, 501, 529, 497, 513, 523, 540, 534, 520, 509, 509, 499, 493, 539, 534, 507, 518, 541, 514, 521, 545, 473, 512] payload_rows_per_dst=[5654, 5498, 5538, 5606, 5560, 5646, 5586, 5584] dup_rows_per_dst=[2644, 2544, 2456, 2690, 2584, 2580, 2688, 2678] combine_rows_per_src=[5654, 5498, 5538, 5606, 5560, 5646, 5586, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=39036 dup_rows=20968 dup_rows_remote=18432 dedup_ratio=0.319946 router_tokens_per_expert=[512, 488, 507, 501, 532, 485, 520, 517, 530, 481, 526, 495, 520, 512, 479, 491, 479, 535, 536, 516, 541, 494, 541, 542, 540, 490, 563, 527, 552, 494, 504, 496, 493, 513, 531, 487, 485, 511, 528, 513, 540, 533, 509, 526, 493, 515, 503, 528, 511, 501, 545, 491, 512, 532, 477, 504, 513, 481, 512, 500, 517, 538, 508, 472] payload_rows_per_dst=[5538, 5532, 5696, 5606, 5520, 5630, 5568, 5478] dup_rows_per_dst=[2586, 2536, 2672, 2726, 2602, 2664, 2578, 2604] combine_rows_per_src=[5538, 5532, 5696, 5606, 5520, 5630, 5568, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=228048 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159169232 ingress_bytes=184107008 link_bytes=184107008 token_bytes=318110416 weight_movement_bytes=25165824 total_nvlink_bytes=343276240 weight_to_token_ratio=0.079110 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400841607e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160202752 dispatch_meta_bytes=229568 combine_payload_bytes=160202752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160432320 ingress_bytes=185368576 link_bytes=185368576 token_bytes=320635072 weight_movement_bytes=25165824 total_nvlink_bytes=345800896 weight_to_token_ratio=0.078487 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410440682e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38846 dup_rows=21076 dup_rows_remote=18490 dedup_ratio=0.321594 router_tokens_per_expert=[523, 528, 497, 529, 513, 535, 518, 474, 540, 528, 534, 495, 518, 478, 538, 503, 501, 504, 511, 517, 507, 525, 528, 572, 509, 514, 538, 478, 531, 523, 511, 496, 458, 501, 494, 480, 512, 501, 477, 496, 539, 514, 510, 525, 533, 500, 477, 502, 517, 539, 519, 500, 500, 532, 527, 512, 514, 531, 486, 493, 505, 522, 512, 524] payload_rows_per_dst=[5572, 5584, 5582, 5614, 5326, 5594, 5596, 5592] dup_rows_per_dst=[2662, 2684, 2748, 2586, 2512, 2606, 2696, 2582] combine_rows_per_src=[5572, 5584, 5582, 5614, 5326, 5594, 5596, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44570 payload_rows_remote=38932 dup_rows=20966 dup_rows_remote=18372 dedup_ratio=0.319916 router_tokens_per_expert=[484, 512, 494, 499, 480, 509, 462, 496, 545, 572, 535, 500, 507, 532, 535, 507, 516, 538, 523, 549, 526, 530, 534, 510, 530, 506, 529, 509, 539, 477, 481, 514, 547, 536, 512, 510, 487, 510, 501, 503, 523, 518, 502, 491, 488, 473, 492, 526, 508, 512, 557, 510, 525, 505, 492, 507, 499, 509, 504, 486, 482, 511, 547, 515] payload_rows_per_dst=[5420, 5650, 5674, 5602, 5516, 5500, 5638, 5570] dup_rows_per_dst=[2452, 2816, 2778, 2568, 2696, 2526, 2594, 2536] combine_rows_per_src=[5420, 5650, 5674, 5602, 5516, 5500, 5638, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=39062 dup_rows=20968 dup_rows_remote=18332 dedup_ratio=0.319946 router_tokens_per_expert=[472, 523, 527, 487, 531, 456, 556, 495, 522, 533, 484, 503, 535, 521, 503, 494, 502, 525, 507, 534, 514, 516, 464, 509, 502, 537, 515, 491, 518, 502, 520, 533, 512, 500, 525, 489, 509, 535, 538, 514, 495, 524, 527, 538, 516, 500, 510, 478, 484, 540, 537, 524, 545, 504, 500, 493, 491, 548, 520, 535, 514, 493, 517, 482] payload_rows_per_dst=[5556, 5598, 5506, 5590, 5564, 5566, 5648, 5540] dup_rows_per_dst=[2538, 2592, 2636, 2646, 2680, 2610, 2606, 2660] combine_rows_per_src=[5556, 5598, 5506, 5590, 5564, 5566, 5648, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38784 dup_rows=21166 dup_rows_remote=18448 dedup_ratio=0.322968 router_tokens_per_expert=[479, 549, 534, 565, 482, 518, 518, 532, 488, 523, 496, 483, 506, 521, 509, 552, 492, 529, 495, 494, 517, 500, 552, 528, 536, 522, 493, 518, 510, 503, 511, 517, 525, 555, 500, 484, 488, 499, 492, 524, 491, 531, 531, 529, 508, 496, 492, 545, 507, 469, 499, 526, 502, 532, 521, 503, 526, 499, 498, 511, 511, 533, 473, 496] payload_rows_per_dst=[5628, 5576, 5262, 5524, 5716, 5620, 5458, 5586] dup_rows_per_dst=[2726, 2464, 2290, 2676, 3080, 2742, 2470, 2718] combine_rows_per_src=[5628, 5576, 5262, 5524, 5716, 5620, 5458, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159891456 dispatch_meta_bytes=229872 combine_payload_bytes=159891456 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160121328 ingress_bytes=210223104 link_bytes=210223104 token_bytes=320012784 weight_movement_bytes=50331648 total_nvlink_bytes=370344432 weight_to_token_ratio=0.157280 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599554922e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44332 payload_rows_remote=38796 dup_rows=21204 dup_rows_remote=18602 dedup_ratio=0.323547 router_tokens_per_expert=[564, 496, 529, 502, 503, 521, 525, 494, 503, 507, 502, 541, 505, 522, 492, 527, 532, 523, 528, 487, 506, 551, 533, 515, 511, 512, 488, 501, 488, 501, 538, 500, 507, 516, 515, 493, 475, 496, 491, 509, 491, 504, 523, 536, 515, 503, 494, 503, 510, 535, 517, 550, 505, 506, 528, 506, 506, 503, 504, 510, 506, 486, 541, 537] payload_rows_per_dst=[5582, 5556, 5630, 5508, 5382, 5536, 5544, 5594] dup_rows_per_dst=[2686, 2642, 2720, 2570, 2622, 2602, 2770, 2592] combine_rows_per_src=[5582, 5556, 5630, 5508, 5382, 5536, 5544, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=229344 combine_payload_bytes=159113216 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159342560 ingress_bytes=234610688 link_bytes=234610688 token_bytes=318455776 weight_movement_bytes=75497472 total_nvlink_bytes=393953248 weight_to_token_ratio=0.237074 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785116258e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159465472 dispatch_meta_bytes=229216 combine_payload_bytes=159465472 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159694688 ingress_bytes=234962944 link_bytes=234962944 token_bytes=319160160 weight_movement_bytes=75497472 total_nvlink_bytes=394657632 weight_to_token_ratio=0.236550 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787796519e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159997952 dispatch_meta_bytes=229576 combine_payload_bytes=159997952 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160227528 ingress_bytes=210329600 link_bytes=210329600 token_bytes=320225480 weight_movement_bytes=50331648 total_nvlink_bytes=370557128 weight_to_token_ratio=0.157176 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600365234e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158859264 dispatch_meta_bytes=228928 combine_payload_bytes=158859264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159088192 ingress_bytes=184025088 link_bytes=184025088 token_bytes=317947456 weight_movement_bytes=25165824 total_nvlink_bytes=343113280 weight_to_token_ratio=0.079151 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400218290e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44714 payload_rows_remote=39066 dup_rows=20822 dup_rows_remote=18178 dedup_ratio=0.317719 router_tokens_per_expert=[550, 503, 514, 482, 511, 497, 542, 539, 516, 534, 513, 519, 536, 517, 521, 466, 539, 529, 493, 515, 495, 530, 546, 485, 546, 495, 494, 519, 506, 509, 481, 480, 496, 478, 581, 515, 493, 488, 524, 513, 476, 542, 491, 531, 536, 508, 516, 457, 486, 504, 474, 517, 532, 545, 511, 521, 514, 501, 512, 502, 539, 508, 515, 520] payload_rows_per_dst=[5648, 5632, 5608, 5500, 5560, 5562, 5602, 5602] dup_rows_per_dst=[2644, 2632, 2740, 2524, 2616, 2552, 2494, 2620] combine_rows_per_src=[5648, 5632, 5608, 5500, 5560, 5562, 5602, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158908416 dispatch_meta_bytes=229592 combine_payload_bytes=158908416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159138008 ingress_bytes=184074240 link_bytes=184074240 token_bytes=318046424 weight_movement_bytes=25165824 total_nvlink_bytes=343212248 weight_to_token_ratio=0.079126 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400592280e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44570 payload_rows_remote=38956 dup_rows=20966 dup_rows_remote=18374 dedup_ratio=0.319916 router_tokens_per_expert=[536, 528, 512, 485, 496, 515, 541, 530, 495, 529, 564, 460, 496, 491, 522, 539, 511, 503, 480, 536, 507, 520, 518, 478, 518, 535, 535, 495, 507, 526, 522, 535, 511, 519, 462, 581, 478, 540, 497, 515, 542, 481, 508, 469, 553, 527, 520, 514, 508, 504, 503, 522, 480, 535, 485, 514, 532, 484, 502, 502, 478, 514, 526, 497] payload_rows_per_dst=[5596, 5568, 5502, 5660, 5614, 5646, 5554, 5430] dup_rows_per_dst=[2690, 2624, 2604, 2686, 2592, 2582, 2548, 2640] combine_rows_per_src=[5596, 5568, 5502, 5660, 5614, 5646, 5554, 5430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44708 payload_rows_remote=39130 dup_rows=20828 dup_rows_remote=18222 dedup_ratio=0.317810 router_tokens_per_expert=[477, 480, 504, 535, 539, 514, 518, 503, 513, 471, 535, 529, 526, 493, 516, 509, 467, 512, 486, 516, 501, 511, 482, 500, 509, 511, 542, 529, 491, 496, 491, 495, 509, 524, 544, 557, 503, 497, 509, 543, 491, 502, 525, 537, 536, 538, 536, 536, 522, 508, 524, 525, 503, 498, 478, 524, 483, 529, 515, 522, 496, 519, 529, 505] payload_rows_per_dst=[5576, 5578, 5500, 5550, 5706, 5592, 5562, 5644] dup_rows_per_dst=[2564, 2606, 2450, 2578, 2666, 2810, 2602, 2552] combine_rows_per_src=[5576, 5578, 5500, 5550, 5706, 5592, 5562, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=228976 combine_payload_bytes=160014336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160243312 ingress_bytes=185180160 link_bytes=185180160 token_bytes=320257648 weight_movement_bytes=25165824 total_nvlink_bytes=345423472 weight_to_token_ratio=0.078580 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409007054e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=229320 combine_payload_bytes=159563776 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159793096 ingress_bytes=260227072 link_bytes=260227072 token_bytes=319356872 weight_movement_bytes=100663296 total_nvlink_bytes=420020168 weight_to_token_ratio=0.315206 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.980027342e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44566 payload_rows_remote=39044 dup_rows=20970 dup_rows_remote=18326 dedup_ratio=0.319977 router_tokens_per_expert=[467, 514, 489, 536, 574, 512, 504, 505, 507, 542, 517, 495, 494, 517, 495, 521, 461, 493, 507, 538, 538, 481, 500, 536, 495, 502, 505, 499, 538, 525, 500, 519, 538, 511, 491, 524, 526, 479, 497, 500, 516, 515, 531, 490, 509, 512, 489, 537, 487, 518, 516, 469, 518, 532, 502, 544, 528, 536, 475, 539, 492, 527, 577, 517] payload_rows_per_dst=[5602, 5594, 5492, 5522, 5504, 5616, 5614, 5622] dup_rows_per_dst=[2600, 2582, 2616, 2644, 2628, 2582, 2558, 2760] combine_rows_per_src=[5602, 5594, 5492, 5522, 5504, 5616, 5614, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44538 payload_rows_remote=38890 dup_rows=20998 dup_rows_remote=18470 dedup_ratio=0.320404 router_tokens_per_expert=[505, 496, 514, 500, 531, 528, 523, 483, 482, 532, 513, 508, 483, 490, 541, 523, 535, 506, 543, 480, 513, 476, 508, 567, 497, 514, 515, 520, 483, 504, 511, 552, 526, 516, 493, 524, 535, 545, 490, 498, 499, 527, 518, 534, 516, 513, 507, 475, 506, 516, 484, 506, 506, 567, 517, 486, 500, 487, 535, 544, 506, 504, 500, 512] payload_rows_per_dst=[5536, 5580, 5576, 5602, 5526, 5510, 5648, 5560] dup_rows_per_dst=[2624, 2564, 2680, 2590, 2728, 2668, 2528, 2616] combine_rows_per_src=[5536, 5580, 5576, 5602, 5526, 5510, 5648, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160276480 dispatch_meta_bytes=229408 combine_payload_bytes=160276480 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160505888 ingress_bytes=210608128 link_bytes=210608128 token_bytes=320782368 weight_movement_bytes=50331648 total_nvlink_bytes=371114016 weight_to_token_ratio=0.156903 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602484510e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44356 payload_rows_remote=38810 dup_rows=21180 dup_rows_remote=18440 dedup_ratio=0.323181 router_tokens_per_expert=[492, 476, 525, 477, 519, 475, 545, 549, 547, 525, 521, 536, 507, 522, 532, 527, 527, 546, 506, 518, 531, 500, 519, 498, 523, 491, 517, 515, 520, 517, 549, 479, 511, 549, 510, 517, 510, 518, 455, 490, 518, 493, 508, 537, 536, 491, 484, 502, 499, 511, 510, 476, 522, 533, 527, 479, 507, 490, 512, 527, 499, 507, 525, 484] payload_rows_per_dst=[5478, 5548, 5628, 5486, 5500, 5500, 5670, 5546] dup_rows_per_dst=[2638, 2510, 2662, 2552, 2574, 2684, 2820, 2740] combine_rows_per_src=[5478, 5548, 5628, 5486, 5500, 5500, 5670, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38952 dup_rows=21050 dup_rows_remote=18472 dedup_ratio=0.321198 router_tokens_per_expert=[535, 504, 532, 515, 506, 498, 512, 489, 503, 490, 525, 484, 524, 518, 521, 485, 538, 550, 471, 502, 507, 519, 473, 496, 499, 532, 523, 540, 519, 509, 515, 558, 518, 524, 499, 515, 473, 498, 515, 521, 506, 522, 538, 514, 498, 486, 559, 529, 532, 523, 535, 507, 526, 517, 522, 450, 502, 502, 524, 489, 487, 523, 506, 516] payload_rows_per_dst=[5542, 5524, 5534, 5682, 5580, 5596, 5534, 5494] dup_rows_per_dst=[2640, 2576, 2578, 2708, 2546, 2708, 2690, 2604] combine_rows_per_src=[5542, 5524, 5534, 5682, 5580, 5596, 5534, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44638 payload_rows_remote=39228 dup_rows=20898 dup_rows_remote=18350 dedup_ratio=0.318878 router_tokens_per_expert=[488, 514, 476, 514, 532, 526, 510, 510, 538, 517, 502, 573, 456, 566, 520, 533, 498, 529, 511, 560, 530, 529, 506, 503, 510, 506, 518, 502, 523, 530, 540, 500, 517, 503, 530, 508, 473, 503, 498, 533, 467, 489, 483, 517, 522, 492, 514, 495, 521, 494, 504, 517, 496, 498, 542, 521, 512, 522, 517, 501, 508, 498, 485, 518] payload_rows_per_dst=[5550, 5664, 5656, 5662, 5572, 5410, 5604, 5520] dup_rows_per_dst=[2590, 2746, 2676, 2596, 2558, 2548, 2582, 2602] combine_rows_per_src=[5550, 5664, 5656, 5662, 5572, 5410, 5604, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=229440 combine_payload_bytes=159293440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159522880 ingress_bytes=184459264 link_bytes=184459264 token_bytes=318816320 weight_movement_bytes=25165824 total_nvlink_bytes=343982144 weight_to_token_ratio=0.078935 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403521868e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159924224 dispatch_meta_bytes=229480 combine_payload_bytes=159924224 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160153704 ingress_bytes=235421696 link_bytes=235421696 token_bytes=320077928 weight_movement_bytes=75497472 total_nvlink_bytes=395575400 weight_to_token_ratio=0.235872 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791287092e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38940 dup_rows=21064 dup_rows_remote=18286 dedup_ratio=0.321411 router_tokens_per_expert=[552, 504, 476, 570, 508, 472, 492, 491, 516, 491, 533, 501, 486, 517, 541, 506, 548, 518, 513, 505, 556, 513, 519, 518, 440, 501, 507, 506, 508, 559, 504, 519, 510, 544, 520, 519, 543, 555, 504, 527, 493, 488, 480, 524, 525, 520, 524, 488, 511, 510, 493, 488, 556, 509, 472, 472, 546, 506, 543, 489, 526, 504, 480, 509] payload_rows_per_dst=[5532, 5636, 5696, 5460, 5638, 5522, 5436, 5552] dup_rows_per_dst=[2778, 2636, 2686, 2558, 2604, 2562, 2586, 2654] combine_rows_per_src=[5532, 5636, 5696, 5460, 5638, 5522, 5436, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158965760 dispatch_meta_bytes=229000 combine_payload_bytes=158965760 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159194760 ingress_bytes=184131584 link_bytes=184131584 token_bytes=318160520 weight_movement_bytes=25165824 total_nvlink_bytes=343326344 weight_to_token_ratio=0.079098 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401028602e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160677888 dispatch_meta_bytes=230312 combine_payload_bytes=160677888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160908200 ingress_bytes=185843712 link_bytes=185843712 token_bytes=321586088 weight_movement_bytes=25165824 total_nvlink_bytes=346751912 weight_to_token_ratio=0.078255 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.414055918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159547392 dispatch_meta_bytes=229696 combine_payload_bytes=159547392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159777088 ingress_bytes=184713216 link_bytes=184713216 token_bytes=319324480 weight_movement_bytes=25165824 total_nvlink_bytes=344490304 weight_to_token_ratio=0.078810 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405454149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44504 payload_rows_remote=38954 dup_rows=21032 dup_rows_remote=18312 dedup_ratio=0.320923 router_tokens_per_expert=[496, 509, 496, 501, 517, 496, 498, 503, 489, 506, 529, 525, 528, 528, 533, 503, 545, 564, 510, 504, 474, 516, 477, 487, 488, 501, 568, 490, 510, 504, 489, 500, 520, 542, 466, 494, 525, 531, 534, 523, 519, 528, 489, 518, 516, 495, 498, 529, 515, 544, 502, 501, 532, 516, 509, 521, 538, 527, 484, 531, 527, 494, 499, 517] payload_rows_per_dst=[5466, 5694, 5566, 5502, 5550, 5572, 5592, 5562] dup_rows_per_dst=[2566, 2588, 2588, 2598, 2720, 2612, 2688, 2672] combine_rows_per_src=[5466, 5694, 5566, 5502, 5550, 5572, 5592, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=228904 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159727144 ingress_bytes=184664064 link_bytes=184664064 token_bytes=319225384 weight_movement_bytes=25165824 total_nvlink_bytes=344391208 weight_to_token_ratio=0.078834 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405080160e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38832 dup_rows=21142 dup_rows_remote=18462 dedup_ratio=0.322601 router_tokens_per_expert=[524, 529, 525, 511, 557, 537, 566, 520, 542, 525, 513, 462, 530, 519, 547, 483, 506, 523, 508, 525, 477, 505, 530, 514, 521, 507, 518, 523, 473, 516, 540, 503, 468, 539, 501, 509, 520, 517, 509, 514, 499, 491, 517, 514, 518, 530, 475, 470, 534, 481, 569, 486, 487, 507, 508, 491, 481, 504, 483, 511, 512, 527, 513, 504] payload_rows_per_dst=[5772, 5562, 5516, 5508, 5482, 5540, 5544, 5470] dup_rows_per_dst=[2766, 2680, 2660, 2694, 2672, 2488, 2582, 2600] combine_rows_per_src=[5772, 5562, 5516, 5508, 5482, 5540, 5544, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=229064 combine_payload_bytes=159555584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159784648 ingress_bytes=184721408 link_bytes=184721408 token_bytes=319340232 weight_movement_bytes=25165824 total_nvlink_bytes=344506056 weight_to_token_ratio=0.078806 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405516481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38952 dup_rows=21046 dup_rows_remote=18430 dedup_ratio=0.321136 router_tokens_per_expert=[501, 489, 550, 511, 537, 507, 495, 507, 492, 529, 543, 512, 523, 501, 515, 525, 504, 502, 539, 512, 522, 530, 517, 515, 525, 489, 493, 487, 518, 473, 533, 540, 515, 544, 488, 494, 521, 475, 511, 514, 521, 535, 497, 503, 503, 539, 500, 563, 525, 530, 481, 514, 505, 502, 515, 505, 520, 493, 494, 504, 500, 508, 488, 525] payload_rows_per_dst=[5636, 5622, 5594, 5458, 5490, 5620, 5538, 5532] dup_rows_per_dst=[2558, 2658, 2688, 2658, 2634, 2702, 2616, 2532] combine_rows_per_src=[5636, 5622, 5594, 5458, 5490, 5620, 5538, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44660 payload_rows_remote=39104 dup_rows=20876 dup_rows_remote=18330 dedup_ratio=0.318542 router_tokens_per_expert=[518, 490, 504, 511, 505, 513, 515, 515, 492, 486, 559, 508, 516, 528, 529, 507, 500, 532, 511, 521, 524, 532, 515, 534, 500, 497, 490, 526, 522, 491, 518, 507, 523, 557, 538, 499, 514, 530, 501, 531, 498, 468, 511, 498, 514, 525, 513, 520, 513, 525, 504, 482, 523, 533, 516, 471, 489, 484, 532, 505, 490, 533, 517, 495] payload_rows_per_dst=[5558, 5610, 5634, 5556, 5642, 5510, 5574, 5576] dup_rows_per_dst=[2584, 2640, 2704, 2546, 2744, 2584, 2560, 2514] combine_rows_per_src=[5558, 5610, 5634, 5556, 5642, 5510, 5574, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159055872 dispatch_meta_bytes=229176 combine_payload_bytes=159055872 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=159285048 ingress_bytes=284884992 link_bytes=284884992 token_bytes=318340920 weight_movement_bytes=125829120 total_nvlink_bytes=444170040 weight_to_token_ratio=0.395265 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.167645623e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38786 dup_rows=21124 dup_rows_remote=18364 dedup_ratio=0.322327 router_tokens_per_expert=[505, 537, 459, 516, 463, 502, 480, 517, 531, 489, 510, 532, 523, 535, 504, 554, 505, 531, 497, 490, 502, 486, 499, 496, 496, 491, 511, 515, 537, 522, 520, 511, 516, 513, 529, 538, 512, 507, 548, 517, 499, 507, 537, 534, 469, 496, 493, 505, 517, 530, 543, 525, 500, 517, 515, 531, 493, 495, 541, 516, 496, 516, 517, 530] payload_rows_per_dst=[5462, 5376, 5494, 5532, 5640, 5644, 5638, 5626] dup_rows_per_dst=[2496, 2396, 2590, 2674, 2720, 2770, 2718, 2760] combine_rows_per_src=[5462, 5376, 5494, 5532, 5640, 5644, 5638, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44616 payload_rows_remote=39140 dup_rows=20920 dup_rows_remote=18262 dedup_ratio=0.319214 router_tokens_per_expert=[524, 483, 509, 498, 514, 522, 497, 518, 520, 507, 532, 473, 538, 527, 531, 528, 494, 521, 492, 513, 500, 502, 493, 518, 531, 519, 537, 530, 496, 481, 537, 544, 519, 508, 519, 525, 466, 490, 542, 512, 476, 519, 502, 529, 483, 510, 548, 500, 507, 537, 492, 501, 524, 491, 517, 511, 521, 523, 479, 506, 518, 495, 547, 522] payload_rows_per_dst=[5604, 5632, 5464, 5642, 5606, 5476, 5622, 5570] dup_rows_per_dst=[2526, 2680, 2602, 2708, 2556, 2658, 2538, 2652] combine_rows_per_src=[5604, 5632, 5464, 5642, 5606, 5476, 5622, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44628 payload_rows_remote=38966 dup_rows=20908 dup_rows_remote=18224 dedup_ratio=0.319031 router_tokens_per_expert=[514, 540, 545, 506, 553, 498, 466, 519, 510, 523, 495, 492, 530, 512, 547, 472, 543, 496, 504, 522, 501, 513, 518, 576, 513, 511, 519, 491, 513, 498, 478, 515, 495, 539, 537, 516, 489, 488, 467, 496, 497, 517, 533, 506, 505, 517, 529, 487, 505, 540, 509, 498, 498, 523, 537, 527, 443, 505, 519, 508, 513, 526, 535, 531] payload_rows_per_dst=[5628, 5520, 5662, 5570, 5578, 5472, 5624, 5574] dup_rows_per_dst=[2654, 2642, 2684, 2506, 2476, 2710, 2650, 2586] combine_rows_per_src=[5628, 5520, 5662, 5570, 5578, 5472, 5624, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160169984 dispatch_meta_bytes=229736 combine_payload_bytes=160169984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160399720 ingress_bytes=185335808 link_bytes=185335808 token_bytes=320569704 weight_movement_bytes=25165824 total_nvlink_bytes=345735528 weight_to_token_ratio=0.078503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410191355e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44530 payload_rows_remote=38914 dup_rows=21006 dup_rows_remote=18362 dedup_ratio=0.320526 router_tokens_per_expert=[561, 515, 501, 504, 490, 487, 513, 492, 531, 527, 525, 505, 492, 529, 507, 559, 515, 460, 528, 516, 515, 490, 513, 508, 532, 507, 521, 486, 522, 517, 532, 537, 452, 496, 525, 511, 517, 495, 497, 516, 482, 502, 476, 543, 523, 503, 526, 534, 538, 496, 493, 529, 505, 490, 511, 516, 530, 519, 533, 520, 519, 515, 473, 546] payload_rows_per_dst=[5616, 5578, 5552, 5598, 5492, 5606, 5516, 5572] dup_rows_per_dst=[2644, 2658, 2538, 2786, 2534, 2572, 2536, 2738] combine_rows_per_src=[5616, 5578, 5552, 5598, 5492, 5606, 5516, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159547392 dispatch_meta_bytes=229528 combine_payload_bytes=159547392 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159776920 ingress_bytes=235044864 link_bytes=235044864 token_bytes=319324312 weight_movement_bytes=75497472 total_nvlink_bytes=394821784 weight_to_token_ratio=0.236429 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788419836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158867456 dispatch_meta_bytes=228600 combine_payload_bytes=158867456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159096056 ingress_bytes=184033280 link_bytes=184033280 token_bytes=317963512 weight_movement_bytes=25165824 total_nvlink_bytes=343129336 weight_to_token_ratio=0.079147 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400280622e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160317440 dispatch_meta_bytes=229608 combine_payload_bytes=160317440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160547048 ingress_bytes=185483264 link_bytes=185483264 token_bytes=320864488 weight_movement_bytes=25165824 total_nvlink_bytes=346030312 weight_to_token_ratio=0.078431 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411313325e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44350 payload_rows_remote=38802 dup_rows=21186 dup_rows_remote=18478 dedup_ratio=0.323273 router_tokens_per_expert=[493, 531, 509, 527, 525, 519, 516, 524, 479, 526, 512, 515, 519, 535, 513, 504, 527, 493, 485, 521, 502, 530, 523, 518, 528, 492, 484, 507, 499, 511, 531, 481, 512, 518, 494, 503, 525, 525, 505, 546, 522, 511, 505, 526, 535, 502, 520, 495, 512, 519, 515, 483, 490, 535, 511, 508, 502, 521, 543, 493, 513, 508, 476, 516] payload_rows_per_dst=[5622, 5532, 5546, 5466, 5548, 5598, 5488, 5550] dup_rows_per_dst=[2666, 2674, 2652, 2600, 2708, 2634, 2658, 2594] combine_rows_per_src=[5622, 5532, 5546, 5466, 5548, 5598, 5488, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159604736 dispatch_meta_bytes=228760 combine_payload_bytes=159604736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159833496 ingress_bytes=184770560 link_bytes=184770560 token_bytes=319438232 weight_movement_bytes=25165824 total_nvlink_bytes=344604056 weight_to_token_ratio=0.078782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405890471e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159391744 dispatch_meta_bytes=229104 combine_payload_bytes=159391744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159620848 ingress_bytes=184557568 link_bytes=184557568 token_bytes=319012592 weight_movement_bytes=25165824 total_nvlink_bytes=344178416 weight_to_token_ratio=0.078887 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404269848e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=38998 dup_rows=21014 dup_rows_remote=18316 dedup_ratio=0.320648 router_tokens_per_expert=[508, 517, 495, 504, 479, 524, 506, 531, 504, 509, 507, 509, 539, 512, 494, 537, 525, 495, 511, 465, 451, 514, 540, 555, 543, 538, 517, 510, 511, 484, 499, 527, 515, 524, 500, 506, 490, 527, 502, 506, 492, 506, 493, 540, 520, 513, 506, 545, 494, 508, 520, 504, 532, 510, 475, 505, 508, 506, 528, 505, 505, 536, 531, 556] payload_rows_per_dst=[5502, 5524, 5592, 5570, 5596, 5582, 5516, 5640] dup_rows_per_dst=[2626, 2698, 2520, 2688, 2544, 2648, 2580, 2710] combine_rows_per_src=[5502, 5524, 5592, 5570, 5596, 5582, 5516, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158932992 dispatch_meta_bytes=229120 combine_payload_bytes=158932992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159162112 ingress_bytes=184098816 link_bytes=184098816 token_bytes=318095104 weight_movement_bytes=25165824 total_nvlink_bytes=343260928 weight_to_token_ratio=0.079114 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400779275e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38894 dup_rows=21002 dup_rows_remote=18244 dedup_ratio=0.320465 router_tokens_per_expert=[519, 517, 541, 501, 537, 470, 497, 524, 475, 523, 483, 545, 490, 548, 504, 521, 495, 487, 509, 535, 545, 494, 520, 528, 528, 508, 559, 490, 496, 557, 543, 518, 517, 492, 467, 500, 513, 524, 531, 512, 510, 539, 478, 488, 507, 486, 523, 503, 485, 514, 504, 507, 515, 540, 496, 504, 514, 531, 530, 497, 521, 504, 499, 510] payload_rows_per_dst=[5572, 5598, 5602, 5640, 5506, 5526, 5556, 5534] dup_rows_per_dst=[2640, 2580, 2624, 2758, 2606, 2542, 2574, 2678] combine_rows_per_src=[5572, 5598, 5602, 5640, 5506, 5526, 5556, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159735808 dispatch_meta_bytes=229256 combine_payload_bytes=159735808 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159965064 ingress_bytes=210067456 link_bytes=210067456 token_bytes=319700872 weight_movement_bytes=50331648 total_nvlink_bytes=370032520 weight_to_token_ratio=0.157434 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598370621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38934 dup_rows=21100 dup_rows_remote=18374 dedup_ratio=0.321960 router_tokens_per_expert=[521, 513, 478, 564, 511, 500, 509, 560, 512, 513, 538, 498, 488, 547, 530, 524, 554, 525, 525, 514, 506, 515, 515, 528, 512, 479, 531, 527, 492, 540, 521, 496, 498, 493, 485, 522, 508, 482, 483, 502, 532, 507, 543, 477, 519, 496, 473, 498, 508, 488, 498, 543, 513, 517, 536, 511, 545, 520, 486, 487, 501, 491, 508, 512] payload_rows_per_dst=[5644, 5636, 5608, 5540, 5432, 5570, 5502, 5504] dup_rows_per_dst=[2668, 2664, 2756, 2656, 2514, 2520, 2726, 2596] combine_rows_per_src=[5644, 5636, 5608, 5540, 5432, 5570, 5502, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44592 payload_rows_remote=39024 dup_rows=20944 dup_rows_remote=18382 dedup_ratio=0.319580 router_tokens_per_expert=[476, 527, 523, 539, 520, 532, 510, 535, 493, 502, 508, 513, 505, 563, 498, 523, 507, 500, 501, 538, 507, 534, 520, 528, 507, 515, 535, 538, 509, 485, 542, 555, 502, 517, 517, 515, 495, 494, 519, 506, 520, 504, 497, 514, 514, 505, 492, 519, 503, 506, 529, 499, 498, 480, 471, 498, 524, 489, 504, 549, 479, 492, 489, 540] payload_rows_per_dst=[5624, 5550, 5594, 5644, 5514, 5568, 5508, 5590] dup_rows_per_dst=[2700, 2660, 2676, 2728, 2616, 2562, 2460, 2542] combine_rows_per_src=[5624, 5550, 5594, 5644, 5514, 5568, 5508, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=39076 dup_rows=20978 dup_rows_remote=18378 dedup_ratio=0.320099 router_tokens_per_expert=[531, 533, 561, 506, 520, 520, 519, 489, 528, 523, 519, 477, 502, 517, 501, 500, 508, 500, 525, 514, 502, 530, 497, 536, 518, 559, 511, 466, 510, 533, 508, 479, 518, 494, 469, 518, 536, 503, 510, 506, 516, 505, 520, 516, 551, 502, 537, 503, 515, 543, 508, 525, 473, 504, 480, 533, 519, 474, 517, 497, 485, 522, 498, 529] payload_rows_per_dst=[5568, 5384, 5544, 5416, 5740, 5748, 5676, 5482] dup_rows_per_dst=[2660, 2426, 2568, 2290, 2830, 2876, 2728, 2600] combine_rows_per_src=[5568, 5384, 5544, 5416, 5740, 5748, 5676, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44566 payload_rows_remote=38962 dup_rows=20970 dup_rows_remote=18396 dedup_ratio=0.319977 router_tokens_per_expert=[510, 450, 510, 485, 510, 505, 505, 523, 538, 544, 543, 510, 523, 508, 490, 519, 490, 512, 517, 517, 491, 514, 502, 546, 499, 518, 505, 541, 522, 498, 522, 522, 484, 516, 551, 496, 491, 491, 533, 490, 501, 507, 524, 520, 505, 488, 516, 500, 494, 535, 512, 494, 564, 484, 541, 501, 507, 538, 524, 530, 537, 524, 506, 475] payload_rows_per_dst=[5492, 5632, 5604, 5568, 5470, 5574, 5636, 5590] dup_rows_per_dst=[2504, 2718, 2574, 2686, 2634, 2548, 2614, 2692] combine_rows_per_src=[5492, 5632, 5604, 5568, 5470, 5574, 5636, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=228552 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159538376 ingress_bytes=234807296 link_bytes=234807296 token_bytes=318848200 weight_movement_bytes=75497472 total_nvlink_bytes=394345672 weight_to_token_ratio=0.236782 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786612218e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44392 payload_rows_remote=38848 dup_rows=21144 dup_rows_remote=18640 dedup_ratio=0.322632 router_tokens_per_expert=[466, 517, 508, 479, 501, 497, 520, 510, 545, 509, 499, 503, 501, 517, 501, 515, 526, 502, 483, 486, 516, 497, 513, 515, 513, 528, 541, 512, 520, 512, 550, 501, 450, 535, 555, 503, 538, 514, 516, 528, 510, 509, 460, 561, 487, 465, 510, 518, 494, 536, 533, 496, 538, 512, 552, 496, 506, 528, 545, 501, 538, 516, 495, 520] payload_rows_per_dst=[5544, 5556, 5500, 5618, 5628, 5440, 5580, 5526] dup_rows_per_dst=[2504, 2704, 2576, 2668, 2650, 2600, 2734, 2708] combine_rows_per_src=[5544, 5556, 5500, 5618, 5628, 5440, 5580, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159473664 dispatch_meta_bytes=229232 combine_payload_bytes=159473664 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159702896 ingress_bytes=184639488 link_bytes=184639488 token_bytes=319176560 weight_movement_bytes=25165824 total_nvlink_bytes=344342384 weight_to_token_ratio=0.078846 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404893165e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159588352 dispatch_meta_bytes=229432 combine_payload_bytes=159588352 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159817784 ingress_bytes=209920000 link_bytes=209920000 token_bytes=319406136 weight_movement_bytes=50331648 total_nvlink_bytes=369737784 weight_to_token_ratio=0.157579 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597248651e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38926 dup_rows=21088 dup_rows_remote=18566 dedup_ratio=0.321777 router_tokens_per_expert=[497, 515, 548, 566, 486, 491, 502, 521, 478, 515, 491, 519, 522, 498, 524, 552, 514, 536, 506, 534, 525, 466, 528, 536, 537, 529, 504, 505, 522, 539, 503, 485, 506, 501, 511, 536, 492, 459, 526, 491, 534, 511, 485, 500, 532, 539, 514, 510, 464, 505, 525, 497, 511, 491, 532, 536, 516, 512, 500, 512, 505, 485, 515, 521] payload_rows_per_dst=[5614, 5528, 5588, 5580, 5522, 5592, 5486, 5538] dup_rows_per_dst=[2638, 2670, 2702, 2668, 2522, 2658, 2636, 2594] combine_rows_per_src=[5614, 5528, 5588, 5580, 5522, 5592, 5486, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159842304 dispatch_meta_bytes=229624 combine_payload_bytes=159842304 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160071928 ingress_bytes=185008128 link_bytes=185008128 token_bytes=319914232 weight_movement_bytes=25165824 total_nvlink_bytes=345080056 weight_to_token_ratio=0.078664 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407698089e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160055296 dispatch_meta_bytes=229816 combine_payload_bytes=160055296 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160285112 ingress_bytes=210386944 link_bytes=210386944 token_bytes=320340408 weight_movement_bytes=50331648 total_nvlink_bytes=370672056 weight_to_token_ratio=0.157119 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600801555e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44572 payload_rows_remote=39082 dup_rows=20964 dup_rows_remote=18412 dedup_ratio=0.319885 router_tokens_per_expert=[537, 518, 539, 526, 499, 522, 499, 502, 492, 512, 493, 491, 524, 479, 529, 501, 521, 497, 525, 528, 531, 501, 494, 450, 498, 479, 483, 509, 485, 519, 505, 484, 529, 533, 556, 500, 533, 494, 585, 537, 499, 520, 515, 519, 504, 514, 524, 507, 511, 508, 531, 490, 521, 532, 528, 513, 520, 491, 524, 519, 518, 501, 528, 492] payload_rows_per_dst=[5576, 5490, 5488, 5478, 5768, 5592, 5600, 5580] dup_rows_per_dst=[2708, 2552, 2606, 2446, 2766, 2612, 2668, 2606] combine_rows_per_src=[5576, 5490, 5488, 5478, 5768, 5592, 5600, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159121408 dispatch_meta_bytes=229952 combine_payload_bytes=159121408 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159351360 ingress_bytes=209453056 link_bytes=209453056 token_bytes=318472768 weight_movement_bytes=50331648 total_nvlink_bytes=368804416 weight_to_token_ratio=0.158041 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593695747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159440896 dispatch_meta_bytes=229968 combine_payload_bytes=159440896 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159670864 ingress_bytes=209772544 link_bytes=209772544 token_bytes=319111760 weight_movement_bytes=50331648 total_nvlink_bytes=369443408 weight_to_token_ratio=0.157724 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596126681e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44588 payload_rows_remote=38912 dup_rows=20948 dup_rows_remote=18288 dedup_ratio=0.319641 router_tokens_per_expert=[485, 523, 456, 565, 531, 522, 532, 511, 488, 536, 516, 504, 505, 498, 489, 499, 517, 529, 510, 543, 543, 517, 520, 494, 523, 539, 511, 527, 507, 520, 534, 507, 499, 504, 511, 517, 503, 517, 487, 536, 501, 488, 510, 503, 522, 498, 504, 522, 522, 474, 495, 508, 498, 536, 504, 498, 504, 493, 519, 518, 488, 517, 549, 522] payload_rows_per_dst=[5554, 5532, 5624, 5676, 5546, 5514, 5530, 5612] dup_rows_per_dst=[2696, 2538, 2722, 2660, 2602, 2582, 2540, 2608] combine_rows_per_src=[5554, 5532, 5624, 5676, 5546, 5514, 5530, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160079872 dispatch_meta_bytes=229976 combine_payload_bytes=160079872 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160309848 ingress_bytes=185245696 link_bytes=185245696 token_bytes=320389720 weight_movement_bytes=25165824 total_nvlink_bytes=345555544 weight_to_token_ratio=0.078548 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409505707e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38890 dup_rows=21052 dup_rows_remote=18562 dedup_ratio=0.321228 router_tokens_per_expert=[521, 543, 489, 544, 479, 493, 514, 501, 509, 525, 523, 521, 507, 529, 547, 553, 487, 504, 521, 497, 525, 526, 472, 499, 497, 517, 546, 502, 480, 481, 529, 514, 516, 505, 526, 531, 487, 554, 538, 467, 491, 505, 509, 470, 486, 521, 549, 531, 510, 503, 534, 518, 504, 511, 497, 465, 557, 483, 506, 487, 553, 508, 549, 502] payload_rows_per_dst=[5560, 5626, 5484, 5532, 5560, 5550, 5594, 5578] dup_rows_per_dst=[2608, 2802, 2578, 2600, 2688, 2574, 2490, 2712] combine_rows_per_src=[5560, 5626, 5484, 5532, 5560, 5550, 5594, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44366 payload_rows_remote=38748 dup_rows=21170 dup_rows_remote=18474 dedup_ratio=0.323029 router_tokens_per_expert=[520, 462, 482, 502, 546, 505, 537, 531, 533, 489, 495, 490, 498, 515, 541, 534, 517, 511, 506, 510, 535, 522, 533, 523, 496, 550, 500, 481, 497, 499, 496, 462, 512, 506, 531, 546, 522, 520, 542, 503, 514, 487, 520, 471, 512, 507, 532, 479, 526, 517, 515, 466, 506, 539, 524, 532, 533, 483, 479, 538, 518, 518, 530, 522] payload_rows_per_dst=[5610, 5532, 5618, 5424, 5608, 5376, 5554, 5644] dup_rows_per_dst=[2560, 2658, 2696, 2538, 2756, 2668, 2696, 2598] combine_rows_per_src=[5610, 5532, 5618, 5424, 5608, 5376, 5554, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38880 dup_rows=21122 dup_rows_remote=18540 dedup_ratio=0.322296 router_tokens_per_expert=[494, 507, 514, 539, 499, 523, 538, 536, 502, 537, 498, 472, 507, 518, 525, 504, 510, 487, 507, 523, 524, 514, 525, 494, 507, 533, 499, 510, 520, 485, 495, 504, 512, 506, 520, 565, 484, 511, 520, 500, 499, 480, 522, 525, 525, 508, 533, 529, 512, 534, 551, 502, 518, 485, 551, 468, 521, 462, 522, 513, 531, 492, 515, 502] payload_rows_per_dst=[5568, 5458, 5576, 5524, 5602, 5558, 5594, 5534] dup_rows_per_dst=[2654, 2668, 2574, 2582, 2712, 2684, 2666, 2582] combine_rows_per_src=[5568, 5458, 5576, 5524, 5602, 5558, 5594, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38880 dup_rows=21036 dup_rows_remote=18388 dedup_ratio=0.320984 router_tokens_per_expert=[486, 468, 533, 514, 522, 523, 490, 504, 495, 506, 544, 505, 512, 490, 489, 492, 549, 533, 476, 494, 522, 514, 511, 489, 538, 538, 529, 499, 485, 472, 520, 521, 494, 530, 537, 488, 500, 507, 494, 525, 512, 578, 507, 473, 531, 530, 496, 507, 512, 533, 533, 539, 487, 498, 493, 524, 481, 530, 529, 525, 533, 519, 541, 519] payload_rows_per_dst=[5526, 5474, 5552, 5542, 5504, 5620, 5580, 5702] dup_rows_per_dst=[2554, 2592, 2624, 2662, 2646, 2648, 2658, 2652] combine_rows_per_src=[5526, 5474, 5552, 5542, 5504, 5620, 5580, 5702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159383552 dispatch_meta_bytes=228800 combine_payload_bytes=159383552 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159612352 ingress_bytes=184549376 link_bytes=184549376 token_bytes=318995904 weight_movement_bytes=25165824 total_nvlink_bytes=344161728 weight_to_token_ratio=0.078891 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404207516e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=229808 combine_payload_bytes=159293440 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159523248 ingress_bytes=209625088 link_bytes=209625088 token_bytes=318816688 weight_movement_bytes=50331648 total_nvlink_bytes=369148336 weight_to_token_ratio=0.157870 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595004711e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44366 payload_rows_remote=38864 dup_rows=21170 dup_rows_remote=18562 dedup_ratio=0.323029 router_tokens_per_expert=[527, 500, 507, 512, 507, 512, 503, 472, 507, 489, 494, 540, 518, 518, 572, 525, 514, 533, 500, 514, 471, 531, 507, 523, 521, 535, 510, 510, 526, 512, 541, 511, 505, 485, 491, 530, 505, 461, 488, 548, 499, 505, 518, 526, 495, 513, 500, 514, 517, 551, 533, 525, 510, 493, 520, 488, 510, 515, 546, 495, 492, 507, 518, 503] payload_rows_per_dst=[5502, 5686, 5480, 5676, 5330, 5660, 5624, 5408] dup_rows_per_dst=[2608, 2730, 2624, 2948, 2404, 2648, 2620, 2588] combine_rows_per_src=[5502, 5686, 5480, 5676, 5330, 5660, 5624, 5408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158711808 dispatch_meta_bytes=228888 combine_payload_bytes=158711808 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158940696 ingress_bytes=234209280 link_bytes=234209280 token_bytes=317652504 weight_movement_bytes=75497472 total_nvlink_bytes=393149976 weight_to_token_ratio=0.237673 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782062007e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44400 payload_rows_remote=38780 dup_rows=21136 dup_rows_remote=18334 dedup_ratio=0.322510 router_tokens_per_expert=[519, 527, 488, 532, 500, 490, 512, 507, 487, 511, 526, 491, 533, 522, 530, 504, 556, 545, 501, 547, 549, 497, 510, 524, 505, 487, 491, 480, 511, 522, 510, 512, 526, 541, 526, 491, 520, 528, 541, 538, 509, 493, 509, 496, 528, 530, 512, 520, 511, 484, 490, 497, 522, 539, 489, 500, 489, 486, 520, 482, 507, 498, 509, 511] payload_rows_per_dst=[5554, 5492, 5698, 5486, 5620, 5594, 5434, 5522] dup_rows_per_dst=[2596, 2716, 2760, 2550, 2802, 2600, 2630, 2482] combine_rows_per_src=[5554, 5492, 5698, 5486, 5620, 5594, 5434, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159252480 dispatch_meta_bytes=229680 combine_payload_bytes=159252480 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159482160 ingress_bytes=209584128 link_bytes=209584128 token_bytes=318734640 weight_movement_bytes=50331648 total_nvlink_bytes=369066288 weight_to_token_ratio=0.157911 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594693053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159252480 dispatch_meta_bytes=229072 combine_payload_bytes=159252480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159481552 ingress_bytes=184418304 link_bytes=184418304 token_bytes=318734032 weight_movement_bytes=25165824 total_nvlink_bytes=343899856 weight_to_token_ratio=0.078956 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403210210e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38856 dup_rows=21058 dup_rows_remote=18294 dedup_ratio=0.321320 router_tokens_per_expert=[511, 524, 468, 512, 488, 495, 514, 447, 522, 504, 508, 507, 542, 530, 549, 531, 531, 519, 524, 540, 520, 535, 483, 486, 506, 532, 514, 510, 495, 540, 522, 537, 527, 487, 526, 496, 515, 483, 487, 521, 475, 508, 488, 491, 520, 521, 536, 507, 527, 509, 531, 506, 495, 496, 517, 486, 535, 511, 507, 565, 508, 498, 509, 534] payload_rows_per_dst=[5486, 5622, 5594, 5606, 5520, 5526, 5516, 5608] dup_rows_per_dst=[2432, 2764, 2682, 2706, 2564, 2566, 2618, 2726] combine_rows_per_src=[5486, 5622, 5594, 5606, 5520, 5526, 5516, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159186944 dispatch_meta_bytes=229704 combine_payload_bytes=159186944 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159416648 ingress_bytes=209518592 link_bytes=209518592 token_bytes=318603592 weight_movement_bytes=50331648 total_nvlink_bytes=368935240 weight_to_token_ratio=0.157976 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594194400e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=228456 combine_payload_bytes=158842880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159071336 ingress_bytes=184008704 link_bytes=184008704 token_bytes=317914216 weight_movement_bytes=25165824 total_nvlink_bytes=343080040 weight_to_token_ratio=0.079159 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400093627e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44778 payload_rows_remote=39230 dup_rows=20758 dup_rows_remote=18224 dedup_ratio=0.316742 router_tokens_per_expert=[539, 539, 469, 500, 506, 499, 527, 497, 477, 509, 491, 492, 525, 559, 521, 503, 507, 486, 546, 516, 555, 476, 462, 509, 525, 521, 521, 516, 509, 474, 491, 484, 466, 513, 511, 486, 517, 510, 507, 543, 497, 525, 479, 533, 533, 547, 524, 526, 493, 483, 531, 534, 541, 489, 559, 522, 519, 531, 493, 498, 538, 523, 526, 520] payload_rows_per_dst=[5558, 5618, 5582, 5548, 5586, 5632, 5606, 5648] dup_rows_per_dst=[2594, 2536, 2532, 2534, 2520, 2696, 2698, 2648] combine_rows_per_src=[5558, 5618, 5582, 5548, 5586, 5632, 5606, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=228600 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159382776 ingress_bytes=209485824 link_bytes=209485824 token_bytes=318536952 weight_movement_bytes=50331648 total_nvlink_bytes=368868600 weight_to_token_ratio=0.158009 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593945073e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18350 dedup_ratio=0.319611 router_tokens_per_expert=[514, 518, 508, 538, 519, 505, 505, 492, 518, 538, 515, 505, 506, 484, 518, 526, 515, 540, 489, 504, 497, 549, 506, 538, 490, 513, 507, 498, 488, 489, 542, 480, 543, 561, 528, 507, 511, 510, 515, 514, 503, 513, 518, 502, 512, 503, 488, 520, 496, 521, 502, 549, 497, 532, 512, 514, 532, 470, 509, 492, 501, 531, 487, 521] payload_rows_per_dst=[5574, 5512, 5608, 5508, 5630, 5574, 5650, 5534] dup_rows_per_dst=[2624, 2708, 2668, 2506, 2748, 2544, 2596, 2552] combine_rows_per_src=[5574, 5512, 5608, 5508, 5630, 5574, 5650, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44498 payload_rows_remote=38918 dup_rows=21038 dup_rows_remote=18432 dedup_ratio=0.321014 router_tokens_per_expert=[499, 497, 505, 502, 501, 529, 499, 528, 504, 484, 498, 529, 491, 501, 495, 541, 483, 511, 561, 529, 491, 478, 488, 552, 531, 507, 509, 522, 510, 563, 508, 480, 519, 508, 480, 547, 484, 472, 464, 550, 533, 489, 520, 547, 479, 503, 521, 529, 566, 527, 478, 490, 547, 529, 525, 479, 516, 554, 522, 486, 517, 508, 530, 523] payload_rows_per_dst=[5472, 5574, 5580, 5570, 5500, 5622, 5612, 5568] dup_rows_per_dst=[2648, 2512, 2606, 2690, 2548, 2620, 2670, 2744] combine_rows_per_src=[5472, 5574, 5580, 5570, 5500, 5622, 5612, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44690 payload_rows_remote=39102 dup_rows=20846 dup_rows_remote=18198 dedup_ratio=0.318085 router_tokens_per_expert=[505, 491, 548, 510, 489, 518, 519, 505, 515, 504, 499, 528, 505, 506, 506, 496, 492, 536, 506, 541, 493, 502, 499, 513, 544, 480, 506, 527, 483, 530, 483, 512, 500, 528, 527, 512, 524, 497, 510, 516, 510, 481, 547, 497, 514, 517, 504, 523, 529, 517, 497, 486, 576, 508, 526, 513, 480, 531, 563, 480, 538, 484, 531, 511] payload_rows_per_dst=[5552, 5534, 5406, 5518, 5660, 5784, 5648, 5588] dup_rows_per_dst=[2618, 2472, 2420, 2612, 2680, 2720, 2676, 2648] combine_rows_per_src=[5552, 5534, 5406, 5518, 5660, 5784, 5648, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44598 payload_rows_remote=39094 dup_rows=20938 dup_rows_remote=18430 dedup_ratio=0.319489 router_tokens_per_expert=[492, 523, 515, 518, 541, 495, 533, 532, 502, 545, 539, 513, 515, 514, 507, 493, 513, 535, 520, 484, 501, 492, 525, 530, 485, 509, 536, 520, 477, 564, 516, 492, 468, 542, 513, 495, 529, 504, 524, 504, 527, 507, 492, 519, 494, 485, 496, 486, 542, 518, 528, 497, 483, 553, 530, 526, 518, 495, 497, 503, 519, 493, 482, 523] payload_rows_per_dst=[5606, 5580, 5556, 5638, 5588, 5504, 5670, 5456] dup_rows_per_dst=[2692, 2676, 2644, 2560, 2570, 2508, 2684, 2604] combine_rows_per_src=[5606, 5580, 5556, 5638, 5588, 5504, 5670, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160686080 dispatch_meta_bytes=229816 combine_payload_bytes=160686080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160915896 ingress_bytes=185851904 link_bytes=185851904 token_bytes=321601976 weight_movement_bytes=25165824 total_nvlink_bytes=346767800 weight_to_token_ratio=0.078251 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.414118249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=229160 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159727400 ingress_bytes=209829888 link_bytes=209829888 token_bytes=319225640 weight_movement_bytes=50331648 total_nvlink_bytes=369557288 weight_to_token_ratio=0.157668 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596563003e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44516 payload_rows_remote=38844 dup_rows=21020 dup_rows_remote=18162 dedup_ratio=0.320740 router_tokens_per_expert=[570, 542, 497, 498, 547, 493, 532, 530, 552, 488, 507, 511, 508, 486, 506, 532, 541, 477, 514, 550, 520, 572, 500, 499, 543, 492, 496, 477, 509, 503, 537, 549, 480, 532, 505, 498, 499, 487, 481, 518, 506, 513, 505, 498, 477, 540, 491, 493, 543, 521, 482, 528, 492, 520, 480, 494, 530, 543, 523, 561, 508, 457, 499, 486] payload_rows_per_dst=[5672, 5566, 5632, 5678, 5482, 5530, 5394, 5562] dup_rows_per_dst=[2858, 2654, 2568, 2826, 2518, 2516, 2434, 2646] combine_rows_per_src=[5672, 5566, 5632, 5678, 5482, 5530, 5394, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160161792 dispatch_meta_bytes=229200 combine_payload_bytes=160161792 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160390992 ingress_bytes=185327616 link_bytes=185327616 token_bytes=320552784 weight_movement_bytes=25165824 total_nvlink_bytes=345718608 weight_to_token_ratio=0.078508 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410129024e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=229400 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=159637528 ingress_bytes=285237248 link_bytes=285237248 token_bytes=319045656 weight_movement_bytes=125829120 total_nvlink_bytes=444874776 weight_to_token_ratio=0.394392 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.170325884e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44392 payload_rows_remote=38826 dup_rows=21144 dup_rows_remote=18520 dedup_ratio=0.322632 router_tokens_per_expert=[493, 527, 510, 511, 523, 499, 523, 495, 517, 471, 545, 542, 507, 497, 522, 519, 523, 494, 512, 511, 502, 529, 486, 546, 541, 497, 488, 550, 509, 495, 518, 535, 551, 517, 519, 481, 518, 518, 520, 471, 495, 495, 481, 490, 527, 514, 517, 513, 512, 467, 545, 501, 492, 573, 523, 502, 481, 502, 544, 524, 528, 457, 552, 501] payload_rows_per_dst=[5578, 5620, 5538, 5564, 5566, 5514, 5502, 5510] dup_rows_per_dst=[2584, 2620, 2668, 2702, 2624, 2550, 2728, 2668] combine_rows_per_src=[5578, 5620, 5538, 5564, 5566, 5514, 5502, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160129024 dispatch_meta_bytes=230096 combine_payload_bytes=160129024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160359120 ingress_bytes=185294848 link_bytes=185294848 token_bytes=320488144 weight_movement_bytes=25165824 total_nvlink_bytes=345653968 weight_to_token_ratio=0.078523 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409879697e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=38984 dup_rows=20988 dup_rows_remote=18326 dedup_ratio=0.320251 router_tokens_per_expert=[494, 532, 507, 484, 514, 482, 514, 526, 509, 465, 543, 534, 538, 505, 529, 490, 479, 504, 534, 498, 506, 517, 502, 510, 508, 539, 526, 512, 494, 493, 540, 521, 535, 475, 473, 494, 527, 544, 528, 528, 512, 541, 482, 503, 539, 535, 507, 494, 520, 507, 517, 498, 544, 521, 553, 507, 483, 466, 487, 520, 539, 515, 508, 517] payload_rows_per_dst=[5596, 5564, 5518, 5608, 5596, 5566, 5642, 5458] dup_rows_per_dst=[2510, 2662, 2582, 2658, 2612, 2660, 2692, 2612] combine_rows_per_src=[5596, 5564, 5518, 5608, 5596, 5566, 5642, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=228024 combine_payload_bytes=159105024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159333048 ingress_bytes=184270848 link_bytes=184270848 token_bytes=318438072 weight_movement_bytes=25165824 total_nvlink_bytes=343603896 weight_to_token_ratio=0.079029 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402088240e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38858 dup_rows=21166 dup_rows_remote=18512 dedup_ratio=0.322968 router_tokens_per_expert=[521, 523, 546, 488, 482, 506, 521, 478, 545, 488, 538, 522, 493, 530, 533, 541, 504, 470, 483, 526, 583, 509, 475, 519, 510, 478, 491, 503, 500, 546, 537, 518, 524, 525, 534, 462, 516, 493, 504, 528, 517, 497, 542, 529, 528, 526, 545, 523, 474, 503, 508, 480, 519, 512, 519, 507, 525, 488, 480, 504, 504, 524, 508, 513] payload_rows_per_dst=[5518, 5614, 5542, 5512, 5494, 5640, 5496, 5554] dup_rows_per_dst=[2612, 2766, 2596, 2654, 2678, 2774, 2548, 2538] combine_rows_per_src=[5518, 5614, 5542, 5512, 5494, 5640, 5496, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159031296 dispatch_meta_bytes=229384 combine_payload_bytes=159031296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159260680 ingress_bytes=184197120 link_bytes=184197120 token_bytes=318291976 weight_movement_bytes=25165824 total_nvlink_bytes=343457800 weight_to_token_ratio=0.079065 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401527255e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159678464 dispatch_meta_bytes=229240 combine_payload_bytes=159678464 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159907704 ingress_bytes=210010112 link_bytes=210010112 token_bytes=319586168 weight_movement_bytes=50331648 total_nvlink_bytes=369917816 weight_to_token_ratio=0.157490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597934299e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44610 payload_rows_remote=38994 dup_rows=20926 dup_rows_remote=18454 dedup_ratio=0.319305 router_tokens_per_expert=[523, 493, 534, 532, 513, 495, 506, 520, 484, 545, 511, 502, 546, 483, 469, 491, 535, 533, 494, 490, 553, 525, 506, 535, 494, 516, 488, 544, 527, 547, 479, 479, 482, 530, 505, 534, 493, 541, 515, 511, 525, 518, 516, 503, 542, 527, 507, 514, 502, 503, 495, 486, 507, 484, 529, 538, 524, 527, 466, 518, 523, 496, 522, 493] payload_rows_per_dst=[5568, 5522, 5556, 5576, 5638, 5626, 5616, 5508] dup_rows_per_dst=[2664, 2540, 2786, 2572, 2584, 2678, 2472, 2630] combine_rows_per_src=[5568, 5522, 5556, 5576, 5638, 5626, 5616, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44730 payload_rows_remote=38962 dup_rows=20806 dup_rows_remote=17816 dedup_ratio=0.317474 router_tokens_per_expert=[485, 550, 491, 527, 479, 504, 500, 530, 494, 507, 490, 505, 524, 504, 503, 525, 493, 534, 519, 491, 525, 513, 511, 552, 496, 538, 510, 494, 543, 550, 510, 489, 518, 485, 515, 479, 493, 493, 508, 499, 473, 521, 531, 469, 499, 488, 514, 504, 528, 529, 510, 562, 535, 550, 493, 520, 536, 530, 551, 481, 490, 518, 537, 523] payload_rows_per_dst=[5598, 5434, 5636, 5442, 5522, 5556, 5774, 5768] dup_rows_per_dst=[2534, 2244, 2636, 2484, 2458, 2542, 2918, 2990] combine_rows_per_src=[5598, 5434, 5636, 5442, 5522, 5556, 5774, 5768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44282 payload_rows_remote=38604 dup_rows=21254 dup_rows_remote=18494 dedup_ratio=0.324310 router_tokens_per_expert=[490, 521, 532, 499, 515, 521, 481, 522, 554, 513, 502, 511, 504, 522, 537, 552, 520, 499, 555, 528, 503, 540, 505, 569, 508, 520, 538, 501, 543, 521, 496, 523, 464, 505, 513, 478, 501, 515, 481, 514, 509, 508, 496, 547, 503, 481, 497, 512, 508, 527, 485, 539, 499, 549, 487, 502, 535, 518, 476, 510, 490, 482, 498, 494] payload_rows_per_dst=[5464, 5648, 5678, 5542, 5454, 5446, 5598, 5452] dup_rows_per_dst=[2698, 2742, 2760, 2758, 2488, 2660, 2594, 2554] combine_rows_per_src=[5464, 5648, 5678, 5542, 5454, 5446, 5598, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159162368 dispatch_meta_bytes=229480 combine_payload_bytes=159162368 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159391848 ingress_bytes=234659840 link_bytes=234659840 token_bytes=318554216 weight_movement_bytes=75497472 total_nvlink_bytes=394051688 weight_to_token_ratio=0.237000 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785490248e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44482 payload_rows_remote=38920 dup_rows=21054 dup_rows_remote=18484 dedup_ratio=0.321259 router_tokens_per_expert=[478, 488, 486, 533, 500, 533, 544, 500, 534, 558, 515, 518, 545, 514, 525, 494, 454, 491, 483, 556, 485, 492, 518, 511, 507, 571, 544, 507, 499, 507, 493, 494, 538, 479, 484, 510, 498, 522, 513, 516, 506, 476, 537, 523, 493, 498, 514, 519, 502, 522, 506, 533, 548, 536, 530, 547, 508, 481, 509, 525, 496, 510, 506, 506] payload_rows_per_dst=[5530, 5686, 5472, 5574, 5544, 5562, 5628, 5486] dup_rows_per_dst=[2594, 2720, 2508, 2670, 2576, 2570, 2820, 2596] combine_rows_per_src=[5530, 5686, 5472, 5574, 5544, 5562, 5628, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159719424 dispatch_meta_bytes=229792 combine_payload_bytes=159719424 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159949216 ingress_bytes=210051072 link_bytes=210051072 token_bytes=319668640 weight_movement_bytes=50331648 total_nvlink_bytes=370000288 weight_to_token_ratio=0.157449 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598245957e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44356 payload_rows_remote=38676 dup_rows=21180 dup_rows_remote=18372 dedup_ratio=0.323181 router_tokens_per_expert=[514, 522, 531, 530, 514, 505, 512, 546, 487, 484, 513, 489, 560, 494, 529, 480, 505, 499, 493, 534, 523, 530, 524, 533, 523, 510, 530, 530, 522, 514, 483, 498, 535, 514, 517, 484, 512, 523, 480, 458, 495, 512, 536, 478, 530, 521, 518, 506, 498, 518, 499, 519, 555, 484, 493, 516, 514, 519, 529, 508, 496, 506, 519, 515] payload_rows_per_dst=[5680, 5562, 5516, 5532, 5696, 5562, 5250, 5558] dup_rows_per_dst=[2808, 2592, 2544, 2688, 3006, 2630, 2258, 2654] combine_rows_per_src=[5680, 5562, 5516, 5532, 5696, 5562, 5250, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159588352 dispatch_meta_bytes=227112 combine_payload_bytes=159588352 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159815464 ingress_bytes=184754176 link_bytes=184754176 token_bytes=319403816 weight_movement_bytes=25165824 total_nvlink_bytes=344569640 weight_to_token_ratio=0.078790 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405765808e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38992 dup_rows=21026 dup_rows_remote=18310 dedup_ratio=0.320831 router_tokens_per_expert=[468, 524, 551, 516, 492, 513, 538, 532, 531, 509, 521, 524, 517, 541, 525, 534, 495, 516, 534, 495, 471, 498, 521, 472, 545, 497, 520, 495, 507, 481, 517, 545, 513, 495, 513, 507, 506, 571, 519, 493, 484, 519, 515, 516, 534, 500, 506, 555, 513, 505, 521, 490, 553, 481, 496, 523, 488, 523, 460, 530, 478, 503, 511, 502] payload_rows_per_dst=[5600, 5644, 5516, 5564, 5518, 5608, 5552, 5508] dup_rows_per_dst=[2668, 2760, 2488, 2650, 2716, 2650, 2612, 2482] combine_rows_per_src=[5600, 5644, 5516, 5564, 5518, 5608, 5552, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158121984 dispatch_meta_bytes=228392 combine_payload_bytes=158121984 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=158350376 ingress_bytes=233619456 link_bytes=233619456 token_bytes=316472360 weight_movement_bytes=75497472 total_nvlink_bytes=391969832 weight_to_token_ratio=0.238559 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.777574128e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44282 payload_rows_remote=38724 dup_rows=21254 dup_rows_remote=18664 dedup_ratio=0.324310 router_tokens_per_expert=[521, 532, 527, 518, 554, 529, 536, 477, 530, 501, 499, 515, 504, 512, 521, 492, 546, 537, 509, 508, 512, 474, 502, 528, 483, 530, 509, 520, 524, 509, 518, 513, 529, 512, 503, 521, 498, 498, 498, 529, 511, 496, 556, 501, 496, 536, 478, 513, 559, 499, 517, 512, 495, 479, 507, 527, 502, 475, 496, 469, 515, 514, 551, 486] payload_rows_per_dst=[5586, 5558, 5534, 5562, 5502, 5528, 5598, 5414] dup_rows_per_dst=[2802, 2590, 2698, 2650, 2674, 2646, 2592, 2602] combine_rows_per_src=[5586, 5558, 5534, 5562, 5502, 5528, 5598, 5414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=229616 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159645936 ingress_bytes=184582144 link_bytes=184582144 token_bytes=319062256 weight_movement_bytes=25165824 total_nvlink_bytes=344228080 weight_to_token_ratio=0.078874 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404456843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158416896 dispatch_meta_bytes=228192 combine_payload_bytes=158416896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158645088 ingress_bytes=183582720 link_bytes=183582720 token_bytes=317061984 weight_movement_bytes=25165824 total_nvlink_bytes=342227808 weight_to_token_ratio=0.079372 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396852381e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38644 dup_rows=21142 dup_rows_remote=18304 dedup_ratio=0.322601 router_tokens_per_expert=[533, 536, 488, 507, 537, 500, 506, 539, 485, 470, 511, 474, 510, 509, 551, 529, 493, 479, 489, 497, 496, 460, 563, 482, 553, 531, 581, 540, 523, 527, 517, 522, 497, 504, 522, 491, 545, 506, 479, 511, 508, 541, 519, 485, 540, 495, 500, 484, 547, 547, 494, 503, 523, 528, 497, 487, 521, 552, 504, 511, 517, 481, 480, 511] payload_rows_per_dst=[5648, 5514, 5364, 5750, 5570, 5494, 5554, 5500] dup_rows_per_dst=[2644, 2564, 2554, 2838, 2540, 2650, 2698, 2654] combine_rows_per_src=[5648, 5514, 5364, 5750, 5570, 5494, 5554, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159711232 dispatch_meta_bytes=229208 combine_payload_bytes=159711232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159940440 ingress_bytes=184877056 link_bytes=184877056 token_bytes=319651672 weight_movement_bytes=25165824 total_nvlink_bytes=344817496 weight_to_token_ratio=0.078729 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406700783e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158613504 dispatch_meta_bytes=229552 combine_payload_bytes=158613504 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158843056 ingress_bytes=183779328 link_bytes=183779328 token_bytes=317456560 weight_movement_bytes=25165824 total_nvlink_bytes=342622384 weight_to_token_ratio=0.079273 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398348341e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38842 dup_rows=21116 dup_rows_remote=18438 dedup_ratio=0.322205 router_tokens_per_expert=[514, 493, 513, 528, 491, 514, 457, 506, 516, 514, 498, 525, 536, 464, 541, 511, 507, 515, 502, 491, 514, 537, 556, 525, 542, 513, 543, 496, 456, 513, 505, 517, 490, 503, 465, 492, 494, 528, 509, 510, 520, 471, 522, 506, 519, 527, 512, 529, 486, 541, 507, 506, 512, 545, 554, 477, 504, 518, 521, 528, 497, 540, 506, 576] payload_rows_per_dst=[5478, 5548, 5560, 5540, 5446, 5592, 5578, 5678] dup_rows_per_dst=[2554, 2662, 2734, 2630, 2536, 2620, 2678, 2702] combine_rows_per_src=[5478, 5548, 5560, 5540, 5446, 5592, 5578, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44644 payload_rows_remote=39094 dup_rows=20892 dup_rows_remote=18306 dedup_ratio=0.318787 router_tokens_per_expert=[529, 523, 498, 498, 526, 503, 524, 500, 516, 490, 549, 564, 561, 515, 481, 515, 506, 506, 521, 512, 513, 537, 516, 525, 501, 499, 483, 540, 470, 510, 496, 516, 543, 524, 485, 549, 490, 523, 483, 516, 499, 526, 530, 525, 539, 505, 511, 479, 477, 480, 548, 504, 508, 516, 510, 501, 507, 515, 511, 493, 509, 498, 509, 512] payload_rows_per_dst=[5582, 5736, 5562, 5502, 5582, 5632, 5498, 5550] dup_rows_per_dst=[2534, 2730, 2710, 2502, 2644, 2596, 2590, 2586] combine_rows_per_src=[5582, 5736, 5562, 5502, 5582, 5632, 5498, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158285824 dispatch_meta_bytes=227792 combine_payload_bytes=158285824 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158513616 ingress_bytes=208617472 link_bytes=208617472 token_bytes=316799440 weight_movement_bytes=50331648 total_nvlink_bytes=367131088 weight_to_token_ratio=0.158875 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.587337918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38824 dup_rows=21116 dup_rows_remote=18516 dedup_ratio=0.322205 router_tokens_per_expert=[522, 521, 506, 555, 504, 493, 506, 476, 503, 518, 469, 493, 486, 516, 475, 520, 511, 530, 505, 523, 508, 502, 510, 509, 478, 477, 527, 501, 502, 513, 513, 467, 495, 503, 498, 509, 525, 499, 512, 529, 522, 536, 511, 537, 518, 535, 499, 540, 528, 526, 499, 559, 530, 501, 477, 595, 524, 491, 521, 541, 510, 495, 524, 540] payload_rows_per_dst=[5556, 5464, 5596, 5472, 5494, 5614, 5628, 5596] dup_rows_per_dst=[2610, 2496, 2600, 2484, 2646, 2782, 2802, 2696] combine_rows_per_src=[5556, 5464, 5596, 5472, 5494, 5614, 5628, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=38960 dup_rows=20982 dup_rows_remote=18438 dedup_ratio=0.320160 router_tokens_per_expert=[533, 511, 515, 490, 510, 528, 508, 535, 523, 526, 524, 506, 512, 514, 519, 490, 551, 498, 487, 491, 482, 525, 513, 540, 491, 503, 491, 503, 501, 505, 482, 518, 500, 538, 516, 526, 485, 489, 491, 512, 484, 514, 516, 506, 515, 516, 516, 502, 530, 528, 495, 526, 530, 484, 504, 532, 497, 519, 510, 519, 547, 510, 538, 548] payload_rows_per_dst=[5570, 5556, 5544, 5530, 5542, 5594, 5542, 5676] dup_rows_per_dst=[2690, 2672, 2630, 2458, 2572, 2544, 2716, 2700] combine_rows_per_src=[5570, 5556, 5544, 5530, 5542, 5594, 5542, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159096832 dispatch_meta_bytes=229120 combine_payload_bytes=159096832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159325952 ingress_bytes=209428480 link_bytes=209428480 token_bytes=318422784 weight_movement_bytes=50331648 total_nvlink_bytes=368754432 weight_to_token_ratio=0.158065 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593508752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160129024 dispatch_meta_bytes=229600 combine_payload_bytes=160129024 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160358624 ingress_bytes=235626496 link_bytes=235626496 token_bytes=320487648 weight_movement_bytes=75497472 total_nvlink_bytes=395985120 weight_to_token_ratio=0.235571 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792845383e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44482 payload_rows_remote=38960 dup_rows=21054 dup_rows_remote=18562 dedup_ratio=0.321259 router_tokens_per_expert=[457, 526, 521, 486, 503, 483, 511, 520, 522, 475, 510, 501, 485, 500, 537, 491, 492, 515, 494, 487, 500, 508, 481, 551, 557, 510, 500, 511, 535, 530, 546, 513, 541, 536, 489, 526, 486, 524, 507, 484, 512, 536, 500, 518, 547, 501, 534, 500, 511, 459, 502, 514, 499, 517, 539, 529, 515, 536, 538, 545, 488, 546, 495, 536] payload_rows_per_dst=[5522, 5514, 5636, 5674, 5658, 5476, 5438, 5564] dup_rows_per_dst=[2492, 2528, 2810, 2730, 2896, 2510, 2460, 2628] combine_rows_per_src=[5522, 5514, 5636, 5674, 5658, 5476, 5438, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38900 dup_rows=21002 dup_rows_remote=18220 dedup_ratio=0.320465 router_tokens_per_expert=[526, 529, 494, 531, 502, 496, 501, 480, 535, 500, 530, 501, 489, 523, 477, 498, 498, 499, 536, 541, 501, 473, 525, 494, 511, 547, 479, 524, 521, 515, 479, 481, 524, 532, 551, 508, 517, 525, 498, 553, 491, 542, 520, 517, 490, 514, 530, 504, 556, 506, 516, 487, 503, 486, 512, 510, 524, 489, 544, 494, 525, 510, 521, 533] payload_rows_per_dst=[5552, 5494, 5548, 5536, 5634, 5552, 5576, 5642] dup_rows_per_dst=[2566, 2612, 2586, 2578, 2782, 2664, 2576, 2638] combine_rows_per_src=[5552, 5494, 5548, 5536, 5634, 5552, 5576, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159023104 dispatch_meta_bytes=229360 combine_payload_bytes=159023104 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159252464 ingress_bytes=184188928 link_bytes=184188928 token_bytes=318275568 weight_movement_bytes=25165824 total_nvlink_bytes=343441392 weight_to_token_ratio=0.079069 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401464924e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38880 dup_rows=21072 dup_rows_remote=18384 dedup_ratio=0.321533 router_tokens_per_expert=[525, 500, 530, 528, 530, 518, 505, 525, 532, 519, 518, 502, 531, 485, 546, 503, 502, 528, 521, 505, 485, 484, 511, 495, 491, 496, 500, 499, 502, 505, 526, 526, 504, 507, 543, 451, 546, 481, 504, 524, 526, 515, 534, 504, 514, 503, 474, 509, 525, 482, 470, 520, 508, 552, 534, 531, 526, 515, 532, 502, 503, 490, 524, 542] payload_rows_per_dst=[5676, 5584, 5492, 5494, 5564, 5500, 5498, 5656] dup_rows_per_dst=[2646, 2688, 2570, 2596, 2556, 2658, 2746, 2612] combine_rows_per_src=[5676, 5584, 5492, 5494, 5564, 5500, 5498, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159580160 dispatch_meta_bytes=229592 combine_payload_bytes=159580160 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159809752 ingress_bytes=209911808 link_bytes=209911808 token_bytes=319389912 weight_movement_bytes=50331648 total_nvlink_bytes=369721560 weight_to_token_ratio=0.157587 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597186319e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44610 payload_rows_remote=38990 dup_rows=20926 dup_rows_remote=18214 dedup_ratio=0.319305 router_tokens_per_expert=[504, 553, 495, 551, 518, 522, 499, 525, 518, 463, 498, 502, 518, 508, 524, 501, 501, 525, 491, 530, 558, 517, 516, 465, 502, 538, 473, 494, 547, 541, 552, 519, 506, 493, 498, 561, 508, 509, 520, 529, 495, 501, 505, 534, 523, 492, 517, 528, 511, 514, 481, 500, 515, 515, 534, 521, 512, 491, 499, 468, 491, 493, 520, 516] payload_rows_per_dst=[5628, 5522, 5564, 5620, 5628, 5590, 5576, 5482] dup_rows_per_dst=[2706, 2542, 2642, 2712, 2620, 2600, 2606, 2498] combine_rows_per_src=[5628, 5522, 5564, 5620, 5628, 5590, 5576, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159580160 dispatch_meta_bytes=230088 combine_payload_bytes=159580160 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159810248 ingress_bytes=209911808 link_bytes=209911808 token_bytes=319390408 weight_movement_bytes=50331648 total_nvlink_bytes=369722056 weight_to_token_ratio=0.157587 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597186319e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=228480 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159562880 ingress_bytes=209666048 link_bytes=209666048 token_bytes=318897280 weight_movement_bytes=50331648 total_nvlink_bytes=369228928 weight_to_token_ratio=0.157830 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595316370e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159252480 dispatch_meta_bytes=229056 combine_payload_bytes=159252480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159481536 ingress_bytes=184418304 link_bytes=184418304 token_bytes=318734016 weight_movement_bytes=25165824 total_nvlink_bytes=343899840 weight_to_token_ratio=0.078956 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403210210e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44600 payload_rows_remote=39112 dup_rows=20936 dup_rows_remote=18450 dedup_ratio=0.319458 router_tokens_per_expert=[452, 515, 548, 534, 484, 528, 521, 500, 526, 526, 519, 507, 546, 515, 478, 490, 526, 544, 496, 509, 511, 520, 503, 516, 479, 510, 553, 472, 537, 517, 480, 504, 528, 506, 539, 500, 532, 488, 471, 506, 522, 509, 477, 532, 525, 535, 481, 541, 487, 493, 503, 484, 475, 532, 502, 511, 523, 498, 508, 544, 553, 538, 539, 520] payload_rows_per_dst=[5578, 5570, 5610, 5584, 5550, 5542, 5488, 5678] dup_rows_per_dst=[2586, 2644, 2640, 2520, 2590, 2702, 2486, 2768] combine_rows_per_src=[5578, 5570, 5610, 5584, 5550, 5542, 5488, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159703040 dispatch_meta_bytes=228816 combine_payload_bytes=159703040 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159931856 ingress_bytes=184868864 link_bytes=184868864 token_bytes=319634896 weight_movement_bytes=25165824 total_nvlink_bytes=344800720 weight_to_token_ratio=0.078733 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406638451e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44446 payload_rows_remote=38904 dup_rows=21090 dup_rows_remote=18552 dedup_ratio=0.321808 router_tokens_per_expert=[493, 492, 500, 511, 536, 487, 479, 480, 510, 511, 476, 522, 514, 494, 512, 475, 529, 493, 563, 499, 511, 497, 508, 519, 537, 529, 522, 498, 528, 534, 558, 545, 487, 509, 485, 514, 520, 534, 536, 524, 497, 541, 493, 512, 483, 546, 537, 546, 552, 507, 540, 481, 522, 487, 525, 494, 510, 530, 504, 508, 496, 513, 486, 487] payload_rows_per_dst=[5328, 5524, 5554, 5724, 5378, 5862, 5534, 5542] dup_rows_per_dst=[2418, 2504, 2598, 2876, 2506, 2930, 2720, 2538] combine_rows_per_src=[5328, 5524, 5554, 5724, 5378, 5862, 5534, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5102483976 prefetch_bytes=352321536 reduce_bytes=276824064 weight_movement_bytes=629145600 egress_bytes=2553076232 ingress_bytes=3178553344 link_bytes=3178553344 total_nvlink_bytes=5731629576 weight_to_token_ratio=0.123302 prefetch_share_of_ingress=0.110843 dedup_ratio=0.321486 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=11 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.511569896e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44512 payload_rows_remote=38934 dup_rows=21024 dup_rows_remote=18286 dedup_ratio=0.320801 router_tokens_per_expert=[531, 510, 559, 538, 487, 509, 508, 514, 504, 490, 499, 500, 506, 512, 490, 505, 502, 512, 537, 528, 499, 520, 524, 536, 503, 497, 505, 491, 475, 517, 496, 477, 521, 515, 505, 532, 525, 526, 517, 553, 539, 529, 481, 521, 523, 492, 488, 511, 521, 495, 540, 514, 539, 530, 486, 524, 511, 514, 540, 482, 522, 515, 483, 493] payload_rows_per_dst=[5632, 5518, 5578, 5426, 5660, 5538, 5660, 5500] dup_rows_per_dst=[2680, 2494, 2738, 2496, 2728, 2630, 2638, 2620] combine_rows_per_src=[5632, 5518, 5578, 5426, 5660, 5538, 5660, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44406 payload_rows_remote=38780 dup_rows=21130 dup_rows_remote=18394 dedup_ratio=0.322418 router_tokens_per_expert=[553, 511, 466, 545, 491, 510, 553, 488, 529, 492, 511, 493, 505, 475, 536, 494, 493, 521, 502, 526, 543, 491, 489, 537, 481, 506, 512, 526, 520, 521, 536, 504, 527, 563, 481, 512, 505, 469, 508, 526, 542, 495, 548, 535, 490, 529, 519, 523, 520, 522, 502, 477, 524, 515, 519, 526, 536, 462, 530, 517, 498, 498, 489, 501] payload_rows_per_dst=[5552, 5512, 5530, 5592, 5552, 5626, 5572, 5470] dup_rows_per_dst=[2682, 2558, 2674, 2620, 2630, 2736, 2638, 2592] combine_rows_per_src=[5552, 5512, 5530, 5592, 5552, 5626, 5572, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5108154576 prefetch_bytes=377487360 reduce_bytes=327155712 weight_movement_bytes=704643072 egress_bytes=2555912400 ingress_bytes=3256885248 link_bytes=3256885248 total_nvlink_bytes=5812797648 weight_to_token_ratio=0.137945 prefetch_share_of_ingress=0.115904 dedup_ratio=0.320927 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=13 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.548820851e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160202752 dispatch_meta_bytes=230248 combine_payload_bytes=160202752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160433000 ingress_bytes=185368576 link_bytes=185368576 token_bytes=320635752 weight_movement_bytes=25165824 total_nvlink_bytes=345801576 weight_to_token_ratio=0.078487 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410440682e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5103920304 prefetch_bytes=327155712 reduce_bytes=603979776 weight_movement_bytes=931135488 egress_bytes=2553791664 ingress_bytes=3481264128 link_bytes=3481264128 total_nvlink_bytes=6035055792 weight_to_token_ratio=0.182435 prefetch_share_of_ingress=0.093976 dedup_ratio=0.320433 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=24 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.655524852e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229824 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159580608 ingress_bytes=184516608 link_bytes=184516608 token_bytes=318931392 weight_movement_bytes=25165824 total_nvlink_bytes=344097216 weight_to_token_ratio=0.078907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403958190e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44310 payload_rows_remote=38772 dup_rows=21226 dup_rows_remote=18458 dedup_ratio=0.323883 router_tokens_per_expert=[497, 530, 495, 521, 512, 532, 525, 541, 493, 510, 498, 479, 495, 504, 493, 483, 552, 506, 541, 489, 505, 473, 480, 560, 454, 540, 502, 498, 516, 522, 534, 515, 520, 515, 502, 525, 517, 489, 517, 494, 527, 483, 512, 496, 525, 530, 531, 529, 521, 516, 516, 515, 546, 521, 485, 491, 523, 507, 511, 563, 511, 507, 490, 538] payload_rows_per_dst=[5538, 5744, 5568, 5572, 5488, 5266, 5584, 5550] dup_rows_per_dst=[2768, 3026, 2644, 2590, 2670, 2140, 2638, 2750] combine_rows_per_src=[5538, 5744, 5568, 5572, 5488, 5266, 5584, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159473664 dispatch_meta_bytes=228880 combine_payload_bytes=159473664 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159702544 ingress_bytes=209805312 link_bytes=209805312 token_bytes=319176208 weight_movement_bytes=50331648 total_nvlink_bytes=369507856 weight_to_token_ratio=0.157692 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596376008e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5112564328 prefetch_bytes=352321536 reduce_bytes=301989888 weight_movement_bytes=654311424 egress_bytes=2558118504 ingress_bytes=3208757248 link_bytes=3208757248 total_nvlink_bytes=5766875752 weight_to_token_ratio=0.127981 prefetch_share_of_ingress=0.109800 dedup_ratio=0.320351 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=12 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.525933446e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=228696 combine_payload_bytes=158842880 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159071576 ingress_bytes=209174528 link_bytes=209174528 token_bytes=317914456 weight_movement_bytes=50331648 total_nvlink_bytes=368246104 weight_to_token_ratio=0.158318 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591576470e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5100068200 prefetch_bytes=327155712 reduce_bytes=327155712 weight_movement_bytes=654311424 egress_bytes=2551864680 ingress_bytes=3202514944 link_bytes=3202514944 total_nvlink_bytes=5754379624 weight_to_token_ratio=0.128295 prefetch_share_of_ingress=0.102156 dedup_ratio=0.321110 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=13 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.522964901e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158810112 dispatch_meta_bytes=228920 combine_payload_bytes=158810112 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159039032 ingress_bytes=259473408 link_bytes=259473408 token_bytes=317849144 weight_movement_bytes=100663296 total_nvlink_bytes=418512440 weight_to_token_ratio=0.316701 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.974292830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5106975192 prefetch_bytes=377487360 reduce_bytes=327155712 weight_movement_bytes=704643072 egress_bytes=2555322840 ingress_bytes=3256295424 link_bytes=3256295424 total_nvlink_bytes=5811618264 weight_to_token_ratio=0.137977 prefetch_share_of_ingress=0.115925 dedup_ratio=0.320988 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=13 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.548540358e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5109466208 prefetch_bytes=352321536 reduce_bytes=176160768 weight_movement_bytes=528482304 egress_bytes=2556568672 ingress_bytes=3081379840 link_bytes=3081379840 total_nvlink_bytes=5637948512 weight_to_token_ratio=0.103432 prefetch_share_of_ingress=0.114339 dedup_ratio=0.320671 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=7 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.465358765e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5097317480 prefetch_bytes=327155712 reduce_bytes=452984832 weight_movement_bytes=780140544 egress_bytes=2550490216 ingress_bytes=3326967808 link_bytes=3326967808 total_nvlink_bytes=5877458024 weight_to_token_ratio=0.153049 prefetch_share_of_ingress=0.098334 dedup_ratio=0.321552 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=18 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.582148808e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none + [2026-09-21 08:32:49] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 211.3 | throughput per GPU (TFLOP/s/GPU): 152.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.026057E+01 | loss scale: 1.0 | grad norm: 28.582 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=39032 dup_rows=21072 dup_rows_remote=18430 dedup_ratio=0.321533 router_tokens_per_expert=[508, 502, 481, 538, 522, 501, 523, 521, 487, 534, 514, 484, 525, 539, 540, 478, 545, 524, 544, 508, 494, 526, 527, 546, 532, 526, 502, 486, 537, 499, 532, 476, 512, 511, 509, 506, 518, 493, 488, 500, 509, 529, 531, 492, 492, 539, 513, 473, 498, 511, 502, 539, 507, 537, 514, 525, 537, 478, 516, 491, 486, 495, 506, 510] payload_rows_per_dst=[5596, 5594, 5718, 5538, 5432, 5558, 5558, 5470] dup_rows_per_dst=[2596, 2608, 2710, 2642, 2642, 2598, 2708, 2568] combine_rows_per_src=[5596, 5594, 5718, 5538, 5432, 5558, 5558, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44382 payload_rows_remote=38684 dup_rows=21154 dup_rows_remote=18504 dedup_ratio=0.322784 router_tokens_per_expert=[485, 497, 515, 541, 491, 448, 532, 556, 521, 530, 531, 478, 495, 508, 513, 508, 562, 555, 472, 530, 497, 505, 530, 523, 488, 493, 519, 471, 512, 522, 511, 517, 532, 503, 509, 510, 515, 481, 537, 532, 501, 504, 534, 541, 499, 514, 515, 477, 462, 533, 493, 543, 500, 518, 525, 525, 486, 500, 491, 567, 492, 545, 524, 504] payload_rows_per_dst=[5562, 5428, 5698, 5474, 5622, 5532, 5570, 5496] dup_rows_per_dst=[2568, 2740, 2650, 2592, 2616, 2638, 2628, 2722] combine_rows_per_src=[5562, 5428, 5698, 5474, 5622, 5532, 5570, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44618 payload_rows_remote=39070 dup_rows=20918 dup_rows_remote=18318 dedup_ratio=0.319183 router_tokens_per_expert=[469, 471, 534, 503, 499, 516, 539, 512, 540, 491, 517, 537, 519, 525, 504, 528, 512, 509, 515, 505, 530, 499, 505, 509, 538, 522, 527, 498, 488, 498, 546, 492, 507, 507, 517, 534, 503, 503, 495, 510, 527, 539, 494, 523, 533, 473, 545, 520, 489, 493, 499, 510, 502, 532, 485, 564, 520, 488, 530, 492, 509, 502, 564, 462] payload_rows_per_dst=[5532, 5704, 5612, 5570, 5542, 5568, 5548, 5542] dup_rows_per_dst=[2554, 2618, 2556, 2648, 2610, 2740, 2600, 2592] combine_rows_per_src=[5532, 5704, 5612, 5570, 5542, 5568, 5548, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44360 payload_rows_remote=38866 dup_rows=21176 dup_rows_remote=18574 dedup_ratio=0.323120 router_tokens_per_expert=[495, 524, 474, 529, 545, 490, 517, 533, 479, 512, 518, 560, 478, 524, 524, 515, 502, 582, 553, 540, 507, 498, 486, 509, 508, 499, 488, 507, 509, 487, 510, 523, 505, 512, 494, 522, 528, 545, 500, 537, 505, 527, 499, 503, 505, 485, 521, 503, 507, 490, 479, 476, 515, 521, 529, 546, 500, 518, 491, 515, 510, 528, 532, 495] payload_rows_per_dst=[5512, 5524, 5620, 5490, 5660, 5494, 5544, 5516] dup_rows_per_dst=[2702, 2696, 2734, 2572, 2626, 2602, 2582, 2662] combine_rows_per_src=[5512, 5524, 5620, 5490, 5660, 5494, 5544, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44452 payload_rows_remote=38888 dup_rows=21084 dup_rows_remote=18412 dedup_ratio=0.321716 router_tokens_per_expert=[525, 517, 503, 530, 524, 517, 480, 503, 469, 522, 487, 491, 511, 479, 562, 528, 543, 476, 502, 485, 508, 502, 492, 536, 489, 533, 546, 507, 520, 515, 522, 486, 493, 531, 496, 507, 522, 540, 475, 516, 508, 497, 516, 464, 546, 528, 516, 536, 487, 518, 518, 521, 511, 526, 513, 481, 488, 518, 540, 584, 500, 493, 543, 526] payload_rows_per_dst=[5606, 5490, 5502, 5564, 5600, 5526, 5552, 5612] dup_rows_per_dst=[2592, 2608, 2586, 2672, 2560, 2696, 2598, 2772] combine_rows_per_src=[5606, 5490, 5502, 5564, 5600, 5526, 5552, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44730 payload_rows_remote=39138 dup_rows=20806 dup_rows_remote=18144 dedup_ratio=0.317474 router_tokens_per_expert=[518, 519, 514, 463, 521, 494, 519, 522, 540, 512, 478, 539, 499, 508, 487, 547, 506, 521, 502, 500, 505, 495, 530, 507, 507, 498, 512, 520, 520, 547, 487, 515, 537, 521, 529, 526, 480, 502, 505, 492, 481, 523, 529, 483, 513, 520, 526, 509, 512, 463, 558, 511, 537, 498, 532, 513, 522, 513, 503, 518, 561, 520, 487, 492] payload_rows_per_dst=[5554, 5624, 5390, 5742, 5612, 5564, 5652, 5592] dup_rows_per_dst=[2586, 2596, 2294, 2822, 2576, 2674, 2596, 2662] combine_rows_per_src=[5554, 5624, 5390, 5742, 5612, 5564, 5652, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44594 payload_rows_remote=39114 dup_rows=20942 dup_rows_remote=18518 dedup_ratio=0.319550 router_tokens_per_expert=[492, 524, 510, 490, 502, 526, 505, 524, 488, 473, 521, 537, 487, 503, 480, 463, 517, 508, 542, 509, 524, 514, 526, 561, 546, 500, 474, 498, 517, 538, 496, 535, 544, 550, 503, 522, 481, 537, 499, 516, 483, 509, 479, 497, 531, 522, 512, 530, 474, 491, 497, 491, 527, 541, 501, 529, 571, 488, 500, 523, 536, 521, 505, 528] payload_rows_per_dst=[5562, 5480, 5726, 5572, 5622, 5590, 5508, 5534] dup_rows_per_dst=[2584, 2424, 2676, 2636, 2682, 2536, 2594, 2810] combine_rows_per_src=[5562, 5480, 5726, 5572, 5622, 5590, 5508, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159875072 dispatch_meta_bytes=229848 combine_payload_bytes=159875072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160104920 ingress_bytes=185040896 link_bytes=185040896 token_bytes=319979992 weight_movement_bytes=25165824 total_nvlink_bytes=345145816 weight_to_token_ratio=0.078648 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407947416e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158449664 dispatch_meta_bytes=228752 combine_payload_bytes=158449664 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=158678416 ingress_bytes=284278784 link_bytes=284278784 token_bytes=317128080 weight_movement_bytes=125829120 total_nvlink_bytes=442957200 weight_to_token_ratio=0.396777 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.163033080e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160030720 dispatch_meta_bytes=229552 combine_payload_bytes=160030720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160260272 ingress_bytes=185196544 link_bytes=185196544 token_bytes=320290992 weight_movement_bytes=25165824 total_nvlink_bytes=345456816 weight_to_token_ratio=0.078572 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409131717e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159195136 dispatch_meta_bytes=229760 combine_payload_bytes=159195136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159424896 ingress_bytes=184360960 link_bytes=184360960 token_bytes=318620032 weight_movement_bytes=25165824 total_nvlink_bytes=343785856 weight_to_token_ratio=0.078984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402773888e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=229200 combine_payload_bytes=159285248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159514448 ingress_bytes=184451072 link_bytes=184451072 token_bytes=318799696 weight_movement_bytes=25165824 total_nvlink_bytes=343965520 weight_to_token_ratio=0.078939 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403459536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160309248 dispatch_meta_bytes=229128 combine_payload_bytes=160309248 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160538376 ingress_bytes=235806720 link_bytes=235806720 token_bytes=320847624 weight_movement_bytes=75497472 total_nvlink_bytes=396345096 weight_to_token_ratio=0.235306 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.794216680e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38776 dup_rows=21118 dup_rows_remote=18330 dedup_ratio=0.322235 router_tokens_per_expert=[515, 502, 506, 546, 516, 497, 518, 529, 492, 494, 514, 557, 513, 506, 500, 500, 510, 571, 498, 528, 520, 524, 501, 505, 481, 490, 509, 534, 481, 518, 503, 485, 508, 491, 531, 500, 540, 495, 505, 497, 521, 555, 463, 475, 536, 538, 498, 524, 494, 515, 490, 493, 519, 552, 520, 495, 487, 519, 514, 526, 520, 509, 535, 540] payload_rows_per_dst=[5642, 5614, 5514, 5456, 5574, 5542, 5540, 5536] dup_rows_per_dst=[2788, 2788, 2550, 2546, 2560, 2678, 2758, 2450] combine_rows_per_src=[5642, 5614, 5514, 5456, 5574, 5542, 5540, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160210944 dispatch_meta_bytes=230528 combine_payload_bytes=160210944 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160441472 ingress_bytes=185376768 link_bytes=185376768 token_bytes=320652416 weight_movement_bytes=25165824 total_nvlink_bytes=345818240 weight_to_token_ratio=0.078483 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410503013e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38922 dup_rows=21114 dup_rows_remote=18604 dedup_ratio=0.322174 router_tokens_per_expert=[526, 532, 538, 489, 526, 505, 510, 519, 509, 527, 540, 492, 515, 491, 552, 497, 534, 464, 534, 494, 528, 497, 486, 535, 510, 510, 490, 515, 510, 514, 529, 545, 512, 487, 489, 476, 528, 530, 485, 498, 499, 458, 551, 479, 522, 503, 486, 510, 499, 512, 534, 517, 550, 519, 496, 506, 502, 532, 580, 500, 482, 484, 549, 530] payload_rows_per_dst=[5594, 5612, 5542, 5554, 5500, 5524, 5538, 5558] dup_rows_per_dst=[2696, 2634, 2602, 2692, 2510, 2492, 2728, 2760] combine_rows_per_src=[5594, 5612, 5542, 5554, 5500, 5524, 5538, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=228424 combine_payload_bytes=158826496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159054920 ingress_bytes=183992320 link_bytes=183992320 token_bytes=317881416 weight_movement_bytes=25165824 total_nvlink_bytes=343047240 weight_to_token_ratio=0.079167 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399968964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38962 dup_rows=21050 dup_rows_remote=18512 dedup_ratio=0.321198 router_tokens_per_expert=[511, 527, 515, 529, 514, 509, 499, 507, 496, 538, 511, 511, 532, 529, 510, 487, 493, 520, 515, 515, 483, 483, 516, 506, 485, 514, 513, 582, 440, 507, 528, 525, 531, 545, 435, 522, 508, 518, 539, 515, 531, 459, 512, 510, 525, 482, 491, 528, 521, 485, 521, 498, 507, 509, 548, 513, 469, 541, 549, 522, 518, 532, 530, 504] payload_rows_per_dst=[5622, 5528, 5524, 5594, 5574, 5448, 5560, 5636] dup_rows_per_dst=[2600, 2700, 2538, 2594, 2652, 2628, 2644, 2694] combine_rows_per_src=[5622, 5528, 5524, 5594, 5574, 5448, 5560, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44674 payload_rows_remote=39108 dup_rows=20862 dup_rows_remote=18366 dedup_ratio=0.318329 router_tokens_per_expert=[508, 526, 497, 545, 499, 482, 492, 478, 518, 524, 496, 539, 515, 523, 490, 536, 531, 509, 548, 512, 476, 507, 507, 484, 532, 530, 540, 501, 535, 490, 487, 515, 536, 496, 480, 551, 515, 520, 555, 527, 511, 479, 514, 495, 504, 535, 481, 512, 503, 527, 483, 518, 499, 504, 532, 505, 474, 546, 498, 544, 528, 502, 507, 515] payload_rows_per_dst=[5494, 5596, 5580, 5622, 5650, 5566, 5570, 5596] dup_rows_per_dst=[2560, 2686, 2568, 2638, 2710, 2496, 2572, 2632] combine_rows_per_src=[5494, 5596, 5580, 5622, 5650, 5566, 5570, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44646 payload_rows_remote=39118 dup_rows=20890 dup_rows_remote=18326 dedup_ratio=0.318756 router_tokens_per_expert=[514, 503, 469, 542, 501, 487, 529, 489, 546, 511, 507, 498, 514, 512, 503, 508, 496, 522, 497, 499, 501, 511, 504, 506, 475, 522, 525, 514, 538, 525, 542, 510, 532, 508, 515, 522, 516, 512, 470, 510, 513, 500, 502, 543, 525, 533, 519, 489, 498, 529, 525, 486, 516, 542, 507, 443, 532, 531, 529, 501, 538, 561, 498, 503] payload_rows_per_dst=[5550, 5642, 5474, 5652, 5514, 5618, 5528, 5668] dup_rows_per_dst=[2518, 2556, 2598, 2650, 2656, 2630, 2564, 2718] combine_rows_per_src=[5550, 5642, 5474, 5652, 5514, 5618, 5528, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44442 payload_rows_remote=38834 dup_rows=21094 dup_rows_remote=18512 dedup_ratio=0.321869 router_tokens_per_expert=[538, 508, 487, 547, 517, 516, 501, 538, 507, 526, 521, 521, 489, 488, 524, 527, 500, 492, 528, 457, 537, 473, 535, 508, 513, 535, 524, 512, 486, 512, 515, 498, 507, 505, 518, 495, 510, 494, 528, 534, 485, 530, 542, 511, 530, 516, 518, 500, 526, 493, 492, 526, 502, 518, 516, 534, 512, 510, 489, 486, 523, 524, 509, 505] payload_rows_per_dst=[5546, 5556, 5504, 5608, 5532, 5618, 5610, 5468] dup_rows_per_dst=[2758, 2650, 2556, 2582, 2650, 2646, 2604, 2648] combine_rows_per_src=[5546, 5556, 5504, 5608, 5532, 5618, 5610, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38936 dup_rows=21052 dup_rows_remote=18408 dedup_ratio=0.321228 router_tokens_per_expert=[485, 524, 533, 531, 514, 513, 501, 507, 525, 535, 522, 493, 472, 489, 533, 549, 511, 504, 501, 517, 515, 520, 517, 505, 487, 497, 503, 541, 532, 502, 527, 459, 553, 511, 474, 545, 502, 519, 530, 504, 547, 513, 481, 485, 523, 527, 526, 495, 511, 531, 507, 506, 513, 528, 506, 538, 459, 529, 488, 562, 504, 511, 466, 510] payload_rows_per_dst=[5386, 5656, 5550, 5446, 5580, 5692, 5626, 5548] dup_rows_per_dst=[2540, 2578, 2630, 2516, 2652, 2792, 2700, 2644] combine_rows_per_src=[5386, 5656, 5550, 5446, 5580, 5692, 5626, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44432 payload_rows_remote=39020 dup_rows=21104 dup_rows_remote=18518 dedup_ratio=0.322021 router_tokens_per_expert=[573, 489, 549, 481, 522, 526, 541, 532, 494, 506, 506, 482, 493, 505, 531, 482, 475, 508, 514, 481, 532, 511, 515, 541, 564, 529, 510, 504, 536, 480, 456, 522, 530, 514, 517, 525, 515, 492, 549, 504, 493, 488, 492, 511, 505, 496, 516, 482, 486, 498, 511, 538, 507, 498, 518, 540, 521, 475, 513, 536, 524, 523, 513, 548] payload_rows_per_dst=[5680, 5412, 5514, 5524, 5676, 5468, 5544, 5614] dup_rows_per_dst=[2746, 2586, 2640, 2678, 2616, 2498, 2648, 2692] combine_rows_per_src=[5680, 5412, 5514, 5524, 5676, 5468, 5544, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=230104 combine_payload_bytes=159424512 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159654616 ingress_bytes=234921984 link_bytes=234921984 token_bytes=319079128 weight_movement_bytes=75497472 total_nvlink_bytes=394576600 weight_to_token_ratio=0.236610 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787484861e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159588352 dispatch_meta_bytes=229896 combine_payload_bytes=159588352 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159818248 ingress_bytes=184754176 link_bytes=184754176 token_bytes=319406600 weight_movement_bytes=25165824 total_nvlink_bytes=344572424 weight_to_token_ratio=0.078789 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405765808e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160186368 dispatch_meta_bytes=229896 combine_payload_bytes=160186368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160416264 ingress_bytes=185352192 link_bytes=185352192 token_bytes=320602632 weight_movement_bytes=25165824 total_nvlink_bytes=345768456 weight_to_token_ratio=0.078495 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410316018e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160227328 dispatch_meta_bytes=229776 combine_payload_bytes=160227328 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160457104 ingress_bytes=185393152 link_bytes=185393152 token_bytes=320684432 weight_movement_bytes=25165824 total_nvlink_bytes=345850256 weight_to_token_ratio=0.078475 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410627677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159064064 dispatch_meta_bytes=229384 combine_payload_bytes=159064064 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159293448 ingress_bytes=209395712 link_bytes=209395712 token_bytes=318357512 weight_movement_bytes=50331648 total_nvlink_bytes=368689160 weight_to_token_ratio=0.158098 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593259425e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159481856 dispatch_meta_bytes=229376 combine_payload_bytes=159481856 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159711232 ingress_bytes=209813504 link_bytes=209813504 token_bytes=319193088 weight_movement_bytes=50331648 total_nvlink_bytes=369524736 weight_to_token_ratio=0.157684 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596438339e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159825920 dispatch_meta_bytes=230152 combine_payload_bytes=159825920 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160056072 ingress_bytes=210157568 link_bytes=210157568 token_bytes=319881992 weight_movement_bytes=50331648 total_nvlink_bytes=370213640 weight_to_token_ratio=0.157344 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599056269e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44532 payload_rows_remote=38904 dup_rows=21004 dup_rows_remote=18386 dedup_ratio=0.320496 router_tokens_per_expert=[485, 518, 515, 522, 514, 509, 477, 512, 521, 526, 506, 539, 508, 518, 524, 501, 545, 492, 519, 509, 515, 496, 470, 503, 467, 526, 479, 482, 562, 498, 519, 540, 516, 504, 502, 505, 516, 512, 506, 491, 531, 532, 531, 517, 566, 500, 485, 548, 545, 495, 496, 528, 494, 476, 512, 504, 525, 479, 522, 520, 498, 534, 518, 543] payload_rows_per_dst=[5628, 5570, 5764, 5596, 5460, 5628, 5522, 5364] dup_rows_per_dst=[2618, 2574, 2952, 2590, 2604, 2792, 2578, 2296] combine_rows_per_src=[5628, 5570, 5764, 5596, 5460, 5628, 5522, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44634 payload_rows_remote=39086 dup_rows=20902 dup_rows_remote=18326 dedup_ratio=0.318939 router_tokens_per_expert=[471, 487, 529, 526, 495, 530, 513, 519, 527, 512, 515, 530, 510, 509, 477, 480, 525, 535, 519, 502, 509, 524, 478, 494, 523, 510, 487, 511, 518, 498, 530, 542, 480, 531, 505, 515, 512, 519, 487, 513, 515, 505, 511, 502, 544, 489, 498, 557, 515, 489, 521, 531, 524, 522, 512, 535, 527, 502, 507, 511, 528, 505, 505, 516] payload_rows_per_dst=[5656, 5578, 5526, 5590, 5548, 5544, 5590, 5602] dup_rows_per_dst=[2484, 2542, 2646, 2648, 2576, 2698, 2708, 2600] combine_rows_per_src=[5656, 5578, 5526, 5590, 5548, 5544, 5590, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38862 dup_rows=21050 dup_rows_remote=18446 dedup_ratio=0.321198 router_tokens_per_expert=[523, 512, 525, 536, 508, 503, 519, 502, 480, 506, 475, 478, 489, 542, 524, 516, 513, 502, 484, 500, 537, 524, 531, 523, 493, 535, 530, 536, 508, 521, 482, 496, 542, 498, 492, 486, 491, 507, 505, 490, 468, 497, 538, 516, 491, 553, 516, 511, 523, 514, 473, 496, 528, 512, 524, 526, 522, 541, 514, 528, 529, 540, 533, 511] payload_rows_per_dst=[5550, 5502, 5624, 5538, 5508, 5542, 5568, 5654] dup_rows_per_dst=[2706, 2518, 2604, 2664, 2514, 2638, 2624, 2782] combine_rows_per_src=[5550, 5502, 5624, 5538, 5508, 5542, 5568, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38880 dup_rows=21118 dup_rows_remote=18400 dedup_ratio=0.322235 router_tokens_per_expert=[509, 520, 499, 543, 503, 525, 500, 499, 536, 496, 549, 517, 508, 503, 488, 487, 486, 519, 536, 516, 476, 515, 530, 481, 489, 536, 487, 497, 528, 506, 537, 478, 506, 536, 500, 535, 523, 493, 499, 532, 513, 544, 511, 529, 505, 523, 478, 525, 522, 486, 539, 493, 489, 521, 541, 491, 522, 513, 500, 518, 515, 507, 533, 527] payload_rows_per_dst=[5546, 5510, 5520, 5494, 5676, 5538, 5554, 5580] dup_rows_per_dst=[2650, 2658, 2598, 2622, 2572, 2718, 2610, 2690] combine_rows_per_src=[5546, 5510, 5520, 5494, 5676, 5538, 5554, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229160 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159579944 ingress_bytes=209682432 link_bytes=209682432 token_bytes=318930728 weight_movement_bytes=50331648 total_nvlink_bytes=369262376 weight_to_token_ratio=0.157814 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595441033e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44544 payload_rows_remote=39004 dup_rows=20992 dup_rows_remote=18498 dedup_ratio=0.320312 router_tokens_per_expert=[523, 507, 560, 477, 526, 493, 522, 521, 540, 523, 538, 494, 497, 544, 532, 500, 511, 493, 503, 492, 522, 526, 482, 486, 510, 497, 512, 500, 517, 492, 480, 509, 544, 482, 488, 517, 537, 527, 552, 512, 509, 508, 473, 499, 519, 511, 541, 546, 512, 523, 515, 493, 524, 494, 545, 540, 480, 515, 494, 516, 506, 496, 498, 523] payload_rows_per_dst=[5620, 5612, 5494, 5540, 5570, 5584, 5650, 5474] dup_rows_per_dst=[2638, 2724, 2536, 2494, 2748, 2628, 2642, 2582] combine_rows_per_src=[5620, 5612, 5494, 5540, 5570, 5584, 5650, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=38868 dup_rows=21012 dup_rows_remote=18308 dedup_ratio=0.320618 router_tokens_per_expert=[487, 483, 488, 513, 508, 531, 563, 480, 528, 518, 507, 534, 500, 485, 501, 512, 519, 496, 500, 522, 505, 531, 546, 502, 502, 489, 517, 508, 561, 495, 533, 512, 478, 532, 498, 488, 509, 494, 526, 547, 554, 484, 541, 512, 523, 502, 495, 555, 540, 539, 500, 535, 505, 520, 528, 513, 502, 520, 492, 516, 498, 516, 445, 485] payload_rows_per_dst=[5562, 5580, 5630, 5572, 5506, 5570, 5656, 5448] dup_rows_per_dst=[2544, 2590, 2612, 2662, 2638, 2762, 2704, 2500] combine_rows_per_src=[5562, 5580, 5630, 5572, 5506, 5570, 5656, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18318 dedup_ratio=0.319611 router_tokens_per_expert=[493, 482, 517, 461, 466, 501, 513, 503, 522, 490, 509, 505, 509, 536, 528, 513, 533, 504, 552, 492, 522, 496, 517, 521, 492, 504, 516, 502, 529, 511, 488, 515, 501, 513, 513, 521, 482, 530, 493, 504, 515, 509, 551, 508, 515, 540, 499, 544, 550, 501, 550, 513, 513, 495, 511, 516, 513, 526, 504, 558, 522, 529, 495, 492] payload_rows_per_dst=[5404, 5638, 5592, 5564, 5558, 5572, 5612, 5650] dup_rows_per_dst=[2468, 2586, 2682, 2550, 2556, 2790, 2686, 2628] combine_rows_per_src=[5404, 5638, 5592, 5564, 5558, 5572, 5612, 5650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38872 dup_rows=21072 dup_rows_remote=18460 dedup_ratio=0.321533 router_tokens_per_expert=[507, 534, 512, 509, 483, 496, 505, 482, 543, 522, 491, 504, 488, 539, 487, 528, 485, 507, 491, 537, 492, 528, 507, 546, 541, 484, 516, 543, 486, 499, 523, 490, 521, 525, 525, 519, 517, 509, 518, 532, 495, 549, 524, 470, 489, 482, 538, 494, 528, 554, 517, 516, 500, 528, 527, 503, 520, 484, 508, 499, 482, 517, 531, 542] payload_rows_per_dst=[5474, 5592, 5566, 5624, 5632, 5416, 5622, 5538] dup_rows_per_dst=[2582, 2612, 2620, 2540, 2700, 2666, 2724, 2628] combine_rows_per_src=[5474, 5592, 5566, 5624, 5632, 5416, 5622, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=160096256 dispatch_meta_bytes=229648 combine_payload_bytes=160096256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160325904 ingress_bytes=185262080 link_bytes=185262080 token_bytes=320422160 weight_movement_bytes=25165824 total_nvlink_bytes=345587984 weight_to_token_ratio=0.078540 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409630370e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159178752 dispatch_meta_bytes=229232 combine_payload_bytes=159178752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159407984 ingress_bytes=184344576 link_bytes=184344576 token_bytes=318586736 weight_movement_bytes=25165824 total_nvlink_bytes=343752560 weight_to_token_ratio=0.078992 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402649225e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159760384 dispatch_meta_bytes=230008 combine_payload_bytes=159760384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159990392 ingress_bytes=184926208 link_bytes=184926208 token_bytes=319750776 weight_movement_bytes=25165824 total_nvlink_bytes=344916600 weight_to_token_ratio=0.078704 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407074772e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159252480 dispatch_meta_bytes=229120 combine_payload_bytes=159252480 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159481600 ingress_bytes=234749952 link_bytes=234749952 token_bytes=318734080 weight_movement_bytes=75497472 total_nvlink_bytes=394231552 weight_to_token_ratio=0.236867 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786175896e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159203328 dispatch_meta_bytes=228704 combine_payload_bytes=159203328 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159432032 ingress_bytes=259866624 link_bytes=259866624 token_bytes=318635360 weight_movement_bytes=100663296 total_nvlink_bytes=419298656 weight_to_token_ratio=0.315920 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.977284749e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=229032 combine_payload_bytes=159498240 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159727272 ingress_bytes=209829888 link_bytes=209829888 token_bytes=319225512 weight_movement_bytes=50331648 total_nvlink_bytes=369557160 weight_to_token_ratio=0.157668 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596563003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159219712 dispatch_meta_bytes=229328 combine_payload_bytes=159219712 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159449040 ingress_bytes=184385536 link_bytes=184385536 token_bytes=318668752 weight_movement_bytes=25165824 total_nvlink_bytes=343834576 weight_to_token_ratio=0.078972 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402960883e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44492 payload_rows_remote=38730 dup_rows=21044 dup_rows_remote=18154 dedup_ratio=0.321106 router_tokens_per_expert=[508, 530, 529, 533, 489, 482, 517, 508, 530, 507, 498, 520, 522, 507, 556, 500, 497, 508, 514, 535, 520, 523, 506, 489, 543, 505, 521, 495, 512, 513, 504, 518, 513, 483, 518, 478, 501, 514, 508, 510, 513, 486, 500, 518, 490, 495, 522, 550, 506, 523, 541, 487, 493, 499, 530, 540, 526, 506, 506, 541, 493, 505, 524, 510] payload_rows_per_dst=[5762, 5574, 5658, 5592, 5590, 5446, 5312, 5558] dup_rows_per_dst=[2890, 2720, 2628, 2842, 2644, 2424, 2274, 2622] combine_rows_per_src=[5762, 5574, 5658, 5592, 5590, 5446, 5312, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158638080 dispatch_meta_bytes=227536 combine_payload_bytes=158638080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158865616 ingress_bytes=183803904 link_bytes=183803904 token_bytes=317503696 weight_movement_bytes=25165824 total_nvlink_bytes=342669520 weight_to_token_ratio=0.079262 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398535336e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44442 payload_rows_remote=38866 dup_rows=21094 dup_rows_remote=18528 dedup_ratio=0.321869 router_tokens_per_expert=[527, 531, 509, 502, 518, 535, 486, 504, 496, 540, 498, 481, 518, 514, 494, 520, 521, 558, 486, 500, 497, 511, 506, 546, 523, 481, 495, 520, 517, 505, 515, 506, 490, 531, 522, 537, 495, 471, 510, 515, 552, 537, 479, 500, 526, 526, 535, 539, 479, 536, 498, 530, 511, 500, 509, 527, 483, 522, 527, 527, 502, 492, 507, 493] payload_rows_per_dst=[5640, 5498, 5542, 5542, 5576, 5614, 5514, 5516] dup_rows_per_dst=[2584, 2624, 2708, 2582, 2566, 2774, 2666, 2590] combine_rows_per_src=[5640, 5498, 5542, 5542, 5576, 5614, 5514, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38990 dup_rows=21026 dup_rows_remote=18534 dedup_ratio=0.320831 router_tokens_per_expert=[471, 510, 526, 512, 517, 510, 492, 534, 537, 507, 501, 505, 546, 559, 509, 505, 504, 483, 505, 517, 493, 496, 500, 508, 495, 520, 497, 500, 543, 495, 511, 511, 517, 488, 513, 536, 483, 516, 496, 478, 469, 528, 505, 531, 569, 527, 552, 499, 512, 528, 504, 527, 529, 491, 548, 526, 526, 512, 522, 523, 518, 480, 475, 521] payload_rows_per_dst=[5506, 5634, 5520, 5500, 5474, 5640, 5654, 5582] dup_rows_per_dst=[2638, 2704, 2492, 2644, 2580, 2720, 2676, 2572] combine_rows_per_src=[5506, 5634, 5520, 5500, 5474, 5640, 5654, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44392 payload_rows_remote=38774 dup_rows=21144 dup_rows_remote=18378 dedup_ratio=0.322632 router_tokens_per_expert=[514, 501, 486, 533, 522, 509, 490, 468, 508, 516, 456, 469, 531, 528, 546, 504, 474, 468, 540, 510, 532, 533, 500, 546, 526, 512, 505, 544, 532, 510, 512, 551, 505, 535, 509, 509, 535, 508, 515, 508, 536, 500, 510, 450, 516, 534, 521, 499, 509, 486, 509, 471, 480, 473, 546, 509, 531, 528, 519, 513, 570, 518, 528, 512] payload_rows_per_dst=[5528, 5502, 5564, 5618, 5600, 5536, 5406, 5638] dup_rows_per_dst=[2518, 2614, 2642, 2766, 2648, 2596, 2560, 2800] combine_rows_per_src=[5528, 5502, 5564, 5618, 5600, 5536, 5406, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38856 dup_rows=21026 dup_rows_remote=18456 dedup_ratio=0.320831 router_tokens_per_expert=[546, 500, 511, 504, 496, 497, 528, 509, 506, 537, 518, 513, 539, 524, 498, 501, 543, 492, 544, 537, 531, 500, 502, 508, 503, 535, 503, 517, 499, 500, 519, 547, 539, 506, 507, 499, 519, 535, 520, 502, 536, 497, 516, 528, 511, 466, 554, 504, 469, 487, 502, 499, 503, 515, 456, 490, 522, 540, 514, 501, 495, 514, 511, 504] payload_rows_per_dst=[5540, 5608, 5584, 5620, 5574, 5654, 5416, 5514] dup_rows_per_dst=[2642, 2664, 2730, 2626, 2680, 2570, 2426, 2688] combine_rows_per_src=[5540, 5608, 5584, 5620, 5574, 5654, 5416, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=39046 dup_rows=21014 dup_rows_remote=18524 dedup_ratio=0.320648 router_tokens_per_expert=[475, 512, 503, 482, 526, 526, 526, 518, 545, 517, 497, 508, 515, 488, 519, 552, 520, 513, 498, 502, 513, 547, 528, 476, 496, 519, 538, 530, 502, 504, 533, 506, 464, 542, 516, 558, 544, 488, 509, 499, 497, 533, 555, 517, 514, 526, 521, 513, 492, 525, 459, 526, 520, 481, 496, 484, 539, 501, 501, 518, 495, 489, 522, 490] payload_rows_per_dst=[5544, 5594, 5494, 5602, 5604, 5616, 5476, 5592] dup_rows_per_dst=[2592, 2688, 2700, 2654, 2636, 2736, 2490, 2518] combine_rows_per_src=[5544, 5594, 5494, 5602, 5604, 5616, 5476, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44366 payload_rows_remote=38808 dup_rows=21170 dup_rows_remote=18650 dedup_ratio=0.323029 router_tokens_per_expert=[491, 508, 534, 537, 520, 526, 526, 520, 485, 500, 514, 537, 475, 521, 515, 528, 522, 499, 504, 488, 519, 505, 550, 491, 489, 523, 544, 523, 499, 537, 531, 511, 523, 510, 505, 522, 528, 524, 505, 504, 539, 496, 489, 470, 539, 505, 518, 531, 523, 516, 506, 523, 521, 498, 486, 495, 514, 515, 517, 513, 472, 469, 496, 524] payload_rows_per_dst=[5578, 5462, 5512, 5538, 5682, 5294, 5742, 5558] dup_rows_per_dst=[2746, 2676, 2644, 2512, 2836, 2280, 2956, 2520] combine_rows_per_src=[5578, 5462, 5512, 5538, 5682, 5294, 5742, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44758 payload_rows_remote=39118 dup_rows=20778 dup_rows_remote=18140 dedup_ratio=0.317047 router_tokens_per_expert=[513, 477, 516, 533, 456, 526, 531, 473, 513, 505, 517, 501, 507, 534, 530, 532, 462, 533, 525, 515, 486, 499, 511, 532, 482, 535, 497, 526, 507, 506, 518, 501, 492, 481, 510, 510, 511, 534, 503, 480, 507, 524, 526, 506, 466, 517, 490, 531, 525, 509, 510, 522, 549, 548, 524, 500, 533, 544, 532, 488, 510, 518, 538, 531] payload_rows_per_dst=[5512, 5640, 5550, 5586, 5558, 5578, 5630, 5704] dup_rows_per_dst=[2538, 2638, 2576, 2558, 2484, 2556, 2744, 2684] combine_rows_per_src=[5512, 5640, 5550, 5586, 5558, 5578, 5630, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159703040 dispatch_meta_bytes=230096 combine_payload_bytes=159703040 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159933136 ingress_bytes=184868864 link_bytes=184868864 token_bytes=319636176 weight_movement_bytes=25165824 total_nvlink_bytes=344802000 weight_to_token_ratio=0.078733 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406638451e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159195136 dispatch_meta_bytes=229576 combine_payload_bytes=159195136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159424712 ingress_bytes=184360960 link_bytes=184360960 token_bytes=318619848 weight_movement_bytes=25165824 total_nvlink_bytes=343785672 weight_to_token_ratio=0.078984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402773888e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158818304 dispatch_meta_bytes=228608 combine_payload_bytes=158818304 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159046912 ingress_bytes=234315776 link_bytes=234315776 token_bytes=317865216 weight_movement_bytes=75497472 total_nvlink_bytes=393362688 weight_to_token_ratio=0.237514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782872318e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229248 combine_payload_bytes=159154176 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159383424 ingress_bytes=234651648 link_bytes=234651648 token_bytes=318537600 weight_movement_bytes=75497472 total_nvlink_bytes=394035072 weight_to_token_ratio=0.237013 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785427916e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159932416 dispatch_meta_bytes=230280 combine_payload_bytes=159932416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160162696 ingress_bytes=185098240 link_bytes=185098240 token_bytes=320095112 weight_movement_bytes=25165824 total_nvlink_bytes=345260936 weight_to_token_ratio=0.078620 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408383737e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158957568 dispatch_meta_bytes=229832 combine_payload_bytes=158957568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159187400 ingress_bytes=184123392 link_bytes=184123392 token_bytes=318144968 weight_movement_bytes=25165824 total_nvlink_bytes=343310792 weight_to_token_ratio=0.079102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400966270e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=38788 dup_rows=21070 dup_rows_remote=18318 dedup_ratio=0.321503 router_tokens_per_expert=[520, 496, 537, 500, 517, 529, 488, 543, 492, 498, 496, 522, 474, 515, 538, 543, 534, 478, 476, 522, 501, 517, 502, 546, 525, 475, 556, 526, 531, 534, 514, 496, 517, 522, 499, 481, 502, 504, 508, 486, 531, 477, 543, 520, 519, 552, 526, 520, 497, 527, 503, 501, 520, 486, 547, 509, 460, 531, 509, 502, 478, 487, 510, 553] payload_rows_per_dst=[5678, 5442, 5594, 5576, 5540, 5562, 5576, 5498] dup_rows_per_dst=[2752, 2548, 2684, 2612, 2498, 2810, 2604, 2562] combine_rows_per_src=[5678, 5442, 5594, 5576, 5540, 5562, 5576, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=160227328 dispatch_meta_bytes=229032 combine_payload_bytes=160227328 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160456360 ingress_bytes=235724800 link_bytes=235724800 token_bytes=320683688 weight_movement_bytes=75497472 total_nvlink_bytes=396181160 weight_to_token_ratio=0.235427 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.793593363e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158875648 dispatch_meta_bytes=228424 combine_payload_bytes=158875648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159104072 ingress_bytes=184041472 link_bytes=184041472 token_bytes=317979720 weight_movement_bytes=25165824 total_nvlink_bytes=343145544 weight_to_token_ratio=0.079143 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400342954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44526 payload_rows_remote=38916 dup_rows=21010 dup_rows_remote=18338 dedup_ratio=0.320587 router_tokens_per_expert=[498, 536, 491, 514, 495, 493, 490, 531, 506, 489, 502, 525, 519, 518, 547, 507, 542, 522, 502, 541, 501, 508, 503, 528, 495, 453, 500, 509, 516, 528, 523, 502, 504, 521, 533, 548, 499, 518, 502, 516, 528, 527, 529, 515, 541, 528, 506, 517, 492, 514, 514, 468, 533, 558, 491, 511, 507, 508, 508, 481, 523, 483, 498, 513] payload_rows_per_dst=[5550, 5534, 5584, 5522, 5610, 5624, 5570, 5532] dup_rows_per_dst=[2546, 2692, 2710, 2530, 2672, 2758, 2592, 2510] combine_rows_per_src=[5550, 5534, 5584, 5522, 5610, 5624, 5570, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44530 payload_rows_remote=38972 dup_rows=21006 dup_rows_remote=18450 dedup_ratio=0.320526 router_tokens_per_expert=[483, 522, 513, 498, 538, 502, 544, 459, 531, 496, 540, 498, 522, 516, 520, 507, 504, 488, 549, 510, 520, 531, 500, 455, 496, 524, 518, 499, 492, 514, 522, 518, 496, 529, 517, 541, 497, 488, 476, 521, 539, 541, 530, 489, 505, 520, 497, 520, 472, 504, 509, 532, 538, 543, 491, 542, 536, 511, 496, 500, 525, 513, 536, 485] payload_rows_per_dst=[5490, 5638, 5558, 5562, 5562, 5596, 5538, 5586] dup_rows_per_dst=[2628, 2622, 2556, 2604, 2568, 2686, 2724, 2618] combine_rows_per_src=[5490, 5638, 5558, 5562, 5562, 5596, 5538, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44432 payload_rows_remote=38818 dup_rows=21104 dup_rows_remote=18442 dedup_ratio=0.322021 router_tokens_per_expert=[544, 517, 501, 510, 508, 490, 544, 519, 500, 556, 529, 498, 509, 513, 504, 487, 526, 504, 499, 514, 486, 492, 510, 539, 498, 497, 522, 505, 529, 520, 525, 542, 479, 488, 481, 509, 499, 523, 576, 500, 499, 503, 519, 470, 515, 501, 547, 496, 506, 532, 562, 509, 560, 487, 506, 512, 513, 513, 505, 491, 500, 533, 514, 483] payload_rows_per_dst=[5586, 5510, 5550, 5614, 5548, 5450, 5676, 5498] dup_rows_per_dst=[2680, 2682, 2590, 2662, 2562, 2650, 2672, 2606] combine_rows_per_src=[5586, 5510, 5550, 5614, 5548, 5450, 5676, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=39024 dup_rows=20968 dup_rows_remote=18350 dedup_ratio=0.319946 router_tokens_per_expert=[540, 474, 482, 518, 492, 510, 537, 521, 545, 504, 509, 505, 532, 518, 511, 488, 531, 510, 493, 520, 519, 523, 493, 503, 524, 519, 511, 541, 511, 530, 470, 561, 501, 533, 502, 502, 469, 540, 526, 533, 492, 511, 519, 501, 504, 508, 532, 514, 509, 513, 500, 481, 528, 487, 507, 518, 499, 484, 531, 535, 486, 514, 532, 512] payload_rows_per_dst=[5550, 5606, 5582, 5616, 5584, 5544, 5468, 5618] dup_rows_per_dst=[2598, 2618, 2602, 2718, 2628, 2618, 2618, 2568] combine_rows_per_src=[5550, 5606, 5582, 5616, 5584, 5544, 5468, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44360 payload_rows_remote=38720 dup_rows=21176 dup_rows_remote=18206 dedup_ratio=0.323120 router_tokens_per_expert=[484, 485, 500, 522, 499, 505, 491, 512, 530, 459, 513, 486, 469, 500, 535, 501, 516, 510, 539, 511, 542, 483, 504, 533, 535, 547, 510, 534, 486, 541, 522, 522, 473, 546, 511, 542, 543, 493, 514, 515, 492, 522, 519, 485, 514, 524, 495, 480, 556, 495, 508, 499, 508, 521, 513, 531, 531, 490, 509, 512, 519, 546, 523, 513] payload_rows_per_dst=[5502, 5450, 5510, 5650, 5630, 5510, 5468, 5640] dup_rows_per_dst=[2494, 2458, 2634, 2774, 2746, 2552, 2548, 2970] combine_rows_per_src=[5502, 5450, 5510, 5650, 5630, 5510, 5468, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44606 payload_rows_remote=38938 dup_rows=20930 dup_rows_remote=18158 dedup_ratio=0.319366 router_tokens_per_expert=[484, 522, 496, 488, 516, 525, 503, 552, 500, 515, 499, 521, 483, 505, 511, 522, 496, 501, 542, 526, 539, 528, 503, 498, 502, 494, 476, 501, 523, 516, 516, 527, 485, 483, 510, 509, 538, 472, 511, 521, 561, 513, 513, 523, 459, 532, 505, 503, 529, 541, 542, 534, 529, 506, 534, 505, 519, 539, 461, 494, 496, 504, 521, 546] payload_rows_per_dst=[5588, 5518, 5682, 5516, 5552, 5592, 5668, 5490] dup_rows_per_dst=[2584, 2594, 2584, 2594, 2506, 2626, 2772, 2670] combine_rows_per_src=[5588, 5518, 5682, 5516, 5552, 5592, 5668, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44202 payload_rows_remote=38614 dup_rows=21334 dup_rows_remote=18626 dedup_ratio=0.325531 router_tokens_per_expert=[507, 521, 503, 512, 508, 533, 511, 509, 546, 507, 534, 524, 522, 488, 515, 512, 544, 504, 518, 508, 518, 541, 510, 502, 508, 508, 499, 513, 487, 493, 527, 505, 480, 483, 531, 517, 517, 501, 555, 482, 496, 507, 545, 531, 511, 499, 518, 507, 528, 489, 519, 442, 530, 506, 536, 515, 509, 509, 527, 497, 503, 536, 493, 512] payload_rows_per_dst=[5522, 5588, 5554, 5478, 5492, 5578, 5468, 5522] dup_rows_per_dst=[2686, 2708, 2736, 2602, 2640, 2650, 2662, 2650] combine_rows_per_src=[5522, 5588, 5554, 5478, 5492, 5578, 5468, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=229688 combine_payload_bytes=159629312 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159859000 ingress_bytes=235126784 link_bytes=235126784 token_bytes=319488312 weight_movement_bytes=75497472 total_nvlink_bytes=394985784 weight_to_token_ratio=0.236307 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789043152e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159399936 dispatch_meta_bytes=229016 combine_payload_bytes=159399936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159628952 ingress_bytes=184565760 link_bytes=184565760 token_bytes=319028888 weight_movement_bytes=25165824 total_nvlink_bytes=344194712 weight_to_token_ratio=0.078883 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404332180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158998528 dispatch_meta_bytes=229040 combine_payload_bytes=158998528 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159227568 ingress_bytes=184164352 link_bytes=184164352 token_bytes=318226096 weight_movement_bytes=25165824 total_nvlink_bytes=343391920 weight_to_token_ratio=0.079082 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401277929e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44542 payload_rows_remote=38830 dup_rows=20994 dup_rows_remote=18174 dedup_ratio=0.320343 router_tokens_per_expert=[513, 513, 528, 498, 525, 536, 531, 512, 525, 544, 502, 545, 483, 535, 515, 493, 500, 512, 495, 531, 522, 479, 504, 504, 539, 510, 507, 483, 515, 483, 502, 508, 498, 511, 526, 518, 527, 545, 469, 524, 507, 534, 501, 511, 496, 512, 523, 489, 542, 512, 519, 542, 502, 488, 508, 543, 486, 524, 517, 508, 516, 496, 511, 471] payload_rows_per_dst=[5712, 5636, 5540, 5556, 5548, 5510, 5482, 5558] dup_rows_per_dst=[2820, 2682, 2564, 2538, 2688, 2592, 2610, 2500] combine_rows_per_src=[5712, 5636, 5540, 5556, 5548, 5510, 5482, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159842304 dispatch_meta_bytes=229496 combine_payload_bytes=159842304 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160071800 ingress_bytes=210173952 link_bytes=210173952 token_bytes=319914104 weight_movement_bytes=50331648 total_nvlink_bytes=370245752 weight_to_token_ratio=0.157329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599180932e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158597120 dispatch_meta_bytes=227704 combine_payload_bytes=158597120 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158824824 ingress_bytes=183762944 link_bytes=183762944 token_bytes=317421944 weight_movement_bytes=25165824 total_nvlink_bytes=342587768 weight_to_token_ratio=0.079282 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398223677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159490048 dispatch_meta_bytes=228384 combine_payload_bytes=159490048 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159718432 ingress_bytes=234987520 link_bytes=234987520 token_bytes=319208480 weight_movement_bytes=75497472 total_nvlink_bytes=394705952 weight_to_token_ratio=0.236515 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787983514e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158162944 dispatch_meta_bytes=228960 combine_payload_bytes=158162944 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158391904 ingress_bytes=208494592 link_bytes=208494592 token_bytes=316554848 weight_movement_bytes=50331648 total_nvlink_bytes=366886496 weight_to_token_ratio=0.158998 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.586402943e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159047680 dispatch_meta_bytes=228016 combine_payload_bytes=159047680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159275696 ingress_bytes=184213504 link_bytes=184213504 token_bytes=318323376 weight_movement_bytes=25165824 total_nvlink_bytes=343489200 weight_to_token_ratio=0.079057 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401651918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44376 payload_rows_remote=38812 dup_rows=21160 dup_rows_remote=18564 dedup_ratio=0.322876 router_tokens_per_expert=[526, 507, 493, 523, 509, 519, 527, 490, 501, 518, 496, 490, 506, 527, 505, 530, 518, 509, 521, 488, 500, 513, 514, 517, 485, 512, 525, 520, 527, 527, 540, 493, 512, 497, 507, 517, 555, 473, 500, 531, 468, 527, 502, 482, 509, 519, 531, 529, 475, 565, 542, 493, 508, 515, 522, 520, 526, 493, 498, 490, 526, 514, 507, 539] payload_rows_per_dst=[5502, 5488, 5564, 5646, 5570, 5542, 5552, 5512] dup_rows_per_dst=[2686, 2658, 2596, 2612, 2614, 2592, 2728, 2674] combine_rows_per_src=[5502, 5488, 5564, 5646, 5570, 5542, 5552, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44498 payload_rows_remote=38946 dup_rows=21038 dup_rows_remote=18384 dedup_ratio=0.321014 router_tokens_per_expert=[532, 548, 514, 468, 514, 497, 512, 500, 509, 488, 532, 520, 501, 497, 497, 519, 499, 513, 490, 494, 520, 561, 502, 492, 525, 513, 509, 507, 488, 522, 530, 512, 468, 523, 540, 556, 518, 557, 473, 468, 542, 509, 545, 515, 514, 485, 525, 517, 494, 515, 529, 566, 495, 530, 514, 487, 494, 510, 480, 507, 530, 505, 538, 494] payload_rows_per_dst=[5494, 5510, 5546, 5584, 5552, 5656, 5590, 5566] dup_rows_per_dst=[2676, 2616, 2596, 2628, 2654, 2648, 2670, 2550] combine_rows_per_src=[5494, 5510, 5546, 5584, 5552, 5656, 5590, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=38890 dup_rows=21014 dup_rows_remote=18388 dedup_ratio=0.320648 router_tokens_per_expert=[527, 521, 524, 508, 553, 493, 492, 559, 483, 522, 520, 555, 528, 501, 512, 508, 522, 498, 472, 484, 484, 487, 506, 508, 493, 544, 516, 515, 531, 493, 528, 509, 501, 505, 496, 504, 491, 503, 496, 530, 504, 494, 549, 515, 533, 531, 525, 535, 515, 510, 518, 498, 486, 485, 522, 535, 534, 504, 534, 509, 509, 502, 521, 478] payload_rows_per_dst=[5592, 5542, 5420, 5632, 5564, 5624, 5600, 5548] dup_rows_per_dst=[2762, 2716, 2502, 2626, 2488, 2748, 2538, 2634] combine_rows_per_src=[5592, 5542, 5420, 5632, 5564, 5624, 5600, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44512 payload_rows_remote=39022 dup_rows=21024 dup_rows_remote=18378 dedup_ratio=0.320801 router_tokens_per_expert=[525, 526, 481, 587, 512, 510, 468, 486, 443, 490, 516, 532, 501, 495, 500, 516, 539, 515, 549, 501, 504, 513, 509, 494, 527, 476, 534, 508, 521, 514, 511, 507, 481, 560, 518, 512, 527, 508, 511, 521, 512, 521, 517, 516, 493, 514, 498, 497, 520, 529, 520, 553, 512, 515, 465, 543, 496, 478, 545, 490, 500, 513, 550, 523] payload_rows_per_dst=[5598, 5456, 5572, 5606, 5622, 5490, 5652, 5516] dup_rows_per_dst=[2592, 2530, 2676, 2590, 2654, 2646, 2662, 2674] combine_rows_per_src=[5598, 5456, 5572, 5606, 5622, 5490, 5652, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38840 dup_rows=21122 dup_rows_remote=18464 dedup_ratio=0.322296 router_tokens_per_expert=[498, 520, 532, 506, 531, 493, 495, 515, 505, 509, 517, 532, 500, 523, 496, 511, 471, 512, 515, 521, 512, 526, 516, 511, 506, 497, 554, 526, 568, 544, 510, 496, 476, 478, 527, 518, 511, 505, 556, 469, 536, 545, 518, 560, 485, 475, 474, 494, 492, 544, 520, 548, 500, 512, 484, 502, 479, 506, 526, 548, 480, 482, 529, 521] payload_rows_per_dst=[5442, 5504, 5624, 5712, 5444, 5560, 5554, 5574] dup_rows_per_dst=[2424, 2682, 2858, 2690, 2616, 2630, 2564, 2658] combine_rows_per_src=[5442, 5504, 5624, 5712, 5444, 5560, 5554, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44474 payload_rows_remote=38950 dup_rows=21062 dup_rows_remote=18470 dedup_ratio=0.321381 router_tokens_per_expert=[486, 531, 523, 518, 520, 467, 528, 533, 497, 472, 559, 528, 506, 481, 526, 491, 521, 518, 532, 528, 494, 517, 527, 505, 505, 495, 525, 525, 489, 494, 488, 494, 504, 525, 525, 505, 521, 505, 516, 508, 563, 508, 473, 501, 531, 534, 538, 533, 502, 491, 513, 525, 511, 503, 478, 535, 473, 540, 480, 525, 506, 515, 505, 553] payload_rows_per_dst=[5574, 5524, 5602, 5494, 5574, 5614, 5524, 5568] dup_rows_per_dst=[2638, 2596, 2682, 2536, 2644, 2748, 2592, 2626] combine_rows_per_src=[5574, 5524, 5602, 5494, 5574, 5614, 5524, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158973952 dispatch_meta_bytes=229504 combine_payload_bytes=158973952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159203456 ingress_bytes=184139776 link_bytes=184139776 token_bytes=318177408 weight_movement_bytes=25165824 total_nvlink_bytes=343343232 weight_to_token_ratio=0.079094 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401090934e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38982 dup_rows=21082 dup_rows_remote=18438 dedup_ratio=0.321686 router_tokens_per_expert=[510, 526, 526, 499, 524, 517, 539, 525, 501, 527, 507, 503, 493, 503, 540, 484, 542, 524, 479, 510, 498, 506, 515, 507, 507, 522, 500, 524, 513, 498, 468, 528, 523, 499, 521, 530, 477, 534, 476, 533, 484, 510, 523, 456, 500, 526, 533, 524, 534, 495, 534, 484, 517, 493, 533, 531, 520, 512, 536, 500, 512, 528, 505, 520] payload_rows_per_dst=[5650, 5472, 5610, 5502, 5600, 5506, 5600, 5514] dup_rows_per_dst=[2682, 2644, 2552, 2618, 2586, 2606, 2642, 2752] combine_rows_per_src=[5650, 5472, 5610, 5502, 5600, 5506, 5600, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=229112 combine_payload_bytes=159293440 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159522552 ingress_bytes=209625088 link_bytes=209625088 token_bytes=318815992 weight_movement_bytes=50331648 total_nvlink_bytes=369147640 weight_to_token_ratio=0.157871 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595004711e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229320 combine_payload_bytes=159522816 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159752136 ingress_bytes=209854464 link_bytes=209854464 token_bytes=319274952 weight_movement_bytes=50331648 total_nvlink_bytes=369606600 weight_to_token_ratio=0.157644 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596749998e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38856 dup_rows=21052 dup_rows_remote=18456 dedup_ratio=0.321228 router_tokens_per_expert=[484, 524, 510, 531, 508, 511, 522, 504, 494, 510, 551, 535, 475, 498, 527, 528, 533, 490, 526, 495, 525, 472, 526, 501, 492, 542, 505, 485, 468, 508, 505, 537, 484, 531, 518, 547, 547, 497, 501, 552, 521, 532, 526, 539, 513, 509, 506, 498, 517, 492, 495, 487, 513, 528, 493, 515, 522, 489, 512, 508, 523, 534, 509, 488] payload_rows_per_dst=[5628, 5646, 5520, 5484, 5674, 5444, 5522, 5566] dup_rows_per_dst=[2596, 2952, 2616, 2576, 2680, 2482, 2546, 2604] combine_rows_per_src=[5628, 5646, 5520, 5484, 5674, 5444, 5522, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159834112 dispatch_meta_bytes=229600 combine_payload_bytes=159834112 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160063712 ingress_bytes=210165760 link_bytes=210165760 token_bytes=319897824 weight_movement_bytes=50331648 total_nvlink_bytes=370229472 weight_to_token_ratio=0.157337 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599118600e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=229216 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159317856 ingress_bytes=184254464 link_bytes=184254464 token_bytes=318406496 weight_movement_bytes=25165824 total_nvlink_bytes=343572320 weight_to_token_ratio=0.079037 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401963577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159539200 dispatch_meta_bytes=229680 combine_payload_bytes=159539200 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159768880 ingress_bytes=235036672 link_bytes=235036672 token_bytes=319308080 weight_movement_bytes=75497472 total_nvlink_bytes=394805552 weight_to_token_ratio=0.236441 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788357504e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229680 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159899952 ingress_bytes=184836096 link_bytes=184836096 token_bytes=319570224 weight_movement_bytes=25165824 total_nvlink_bytes=344736048 weight_to_token_ratio=0.078749 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406389124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229248 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159383424 ingress_bytes=209485824 link_bytes=209485824 token_bytes=318537600 weight_movement_bytes=50331648 total_nvlink_bytes=368869248 weight_to_token_ratio=0.158008 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593945073e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44364 payload_rows_remote=38876 dup_rows=21172 dup_rows_remote=18576 dedup_ratio=0.323059 router_tokens_per_expert=[518, 485, 524, 544, 549, 521, 492, 500, 530, 496, 516, 519, 476, 480, 500, 510, 483, 508, 515, 543, 505, 503, 490, 495, 482, 533, 508, 507, 532, 531, 526, 507, 498, 500, 502, 538, 515, 533, 522, 553, 494, 513, 539, 549, 499, 509, 512, 522, 512, 497, 526, 467, 503, 529, 484, 513, 532, 468, 498, 511, 517, 508, 519, 558] payload_rows_per_dst=[5562, 5492, 5488, 5616, 5558, 5540, 5550, 5558] dup_rows_per_dst=[2704, 2562, 2596, 2636, 2764, 2734, 2512, 2664] combine_rows_per_src=[5562, 5492, 5488, 5616, 5558, 5540, 5550, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44476 payload_rows_remote=38960 dup_rows=21060 dup_rows_remote=18372 dedup_ratio=0.321350 router_tokens_per_expert=[519, 523, 506, 517, 513, 521, 501, 536, 504, 521, 508, 487, 569, 487, 472, 491, 515, 482, 509, 518, 489, 526, 506, 518, 546, 476, 538, 477, 494, 485, 556, 530, 526, 505, 497, 522, 504, 521, 513, 511, 550, 474, 526, 467, 485, 528, 508, 527, 560, 511, 505, 492, 510, 501, 528, 532, 517, 503, 515, 532, 528, 513, 518, 499] payload_rows_per_dst=[5626, 5474, 5516, 5516, 5568, 5572, 5620, 5584] dup_rows_per_dst=[2646, 2604, 2610, 2688, 2630, 2558, 2658, 2666] combine_rows_per_src=[5626, 5474, 5516, 5516, 5568, 5572, 5620, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39084 dup_rows=20934 dup_rows_remote=18324 dedup_ratio=0.319427 router_tokens_per_expert=[496, 512, 519, 489, 535, 524, 495, 509, 534, 492, 511, 531, 533, 513, 509, 497, 520, 556, 505, 524, 526, 537, 487, 507, 514, 494, 532, 493, 502, 508, 510, 500, 518, 487, 492, 528, 494, 546, 506, 493, 522, 520, 515, 512, 519, 537, 499, 526, 518, 482, 520, 495, 484, 533, 506, 507, 539, 480, 528, 530, 524, 498, 487, 509] payload_rows_per_dst=[5576, 5624, 5666, 5534, 5518, 5638, 5518, 5528] dup_rows_per_dst=[2582, 2616, 2658, 2572, 2610, 2662, 2572, 2662] combine_rows_per_src=[5576, 5624, 5666, 5534, 5518, 5638, 5518, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159236096 dispatch_meta_bytes=229808 combine_payload_bytes=159236096 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159465904 ingress_bytes=184401920 link_bytes=184401920 token_bytes=318702000 weight_movement_bytes=25165824 total_nvlink_bytes=343867824 weight_to_token_ratio=0.078963 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403085547e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44474 payload_rows_remote=38820 dup_rows=21062 dup_rows_remote=18380 dedup_ratio=0.321381 router_tokens_per_expert=[493, 520, 522, 527, 499, 528, 508, 509, 509, 541, 528, 537, 488, 522, 524, 519, 529, 538, 498, 494, 487, 548, 485, 503, 532, 515, 486, 537, 543, 549, 518, 498, 495, 528, 495, 495, 496, 503, 446, 546, 521, 504, 503, 518, 490, 471, 528, 496, 518, 522, 531, 518, 535, 515, 483, 546, 520, 493, 527, 497, 501, 514, 486, 493] payload_rows_per_dst=[5548, 5642, 5540, 5630, 5486, 5466, 5654, 5508] dup_rows_per_dst=[2664, 2694, 2624, 2726, 2522, 2596, 2682, 2554] combine_rows_per_src=[5548, 5642, 5540, 5630, 5486, 5466, 5654, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44530 payload_rows_remote=39048 dup_rows=21006 dup_rows_remote=18466 dedup_ratio=0.320526 router_tokens_per_expert=[537, 496, 469, 469, 492, 481, 508, 546, 465, 514, 493, 539, 489, 486, 529, 482, 516, 502, 524, 532, 482, 498, 513, 501, 526, 519, 530, 546, 531, 519, 535, 528, 488, 521, 495, 488, 509, 538, 523, 550, 509, 491, 489, 522, 524, 504, 489, 483, 515, 531, 487, 565, 523, 515, 540, 520, 495, 562, 531, 531, 497, 471, 540, 525] payload_rows_per_dst=[5504, 5454, 5542, 5694, 5580, 5482, 5622, 5652] dup_rows_per_dst=[2492, 2540, 2594, 2774, 2644, 2540, 2770, 2652] combine_rows_per_src=[5504, 5454, 5542, 5694, 5580, 5482, 5622, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38934 dup_rows=21058 dup_rows_remote=18336 dedup_ratio=0.321320 router_tokens_per_expert=[537, 513, 456, 468, 529, 510, 458, 493, 487, 520, 551, 515, 478, 519, 518, 500, 519, 502, 500, 511, 507, 499, 484, 507, 482, 522, 522, 527, 525, 543, 489, 456, 518, 553, 508, 515, 525, 520, 532, 515, 544, 460, 544, 534, 544, 491, 491, 520, 509, 513, 517, 526, 513, 532, 561, 505, 514, 530, 540, 477, 511, 527, 515, 517] payload_rows_per_dst=[5414, 5586, 5446, 5562, 5664, 5592, 5670, 5544] dup_rows_per_dst=[2514, 2590, 2612, 2570, 2704, 2664, 2682, 2722] combine_rows_per_src=[5414, 5586, 5446, 5562, 5664, 5592, 5670, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159580160 dispatch_meta_bytes=229328 combine_payload_bytes=159580160 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159809488 ingress_bytes=209911808 link_bytes=209911808 token_bytes=319389648 weight_movement_bytes=50331648 total_nvlink_bytes=369721296 weight_to_token_ratio=0.157587 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597186319e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44520 payload_rows_remote=38982 dup_rows=21016 dup_rows_remote=18472 dedup_ratio=0.320679 router_tokens_per_expert=[509, 523, 502, 492, 489, 540, 507, 526, 479, 501, 507, 493, 492, 530, 497, 542, 493, 507, 535, 521, 550, 499, 489, 449, 530, 527, 533, 534, 499, 493, 542, 514, 505, 508, 492, 540, 514, 524, 522, 543, 503, 527, 525, 517, 526, 487, 504, 487, 518, 502, 527, 505, 499, 501, 485, 557, 527, 507, 537, 501, 517, 485, 516, 516] payload_rows_per_dst=[5584, 5538, 5478, 5636, 5580, 5538, 5596, 5570] dup_rows_per_dst=[2592, 2544, 2608, 2708, 2716, 2614, 2592, 2642] combine_rows_per_src=[5584, 5538, 5478, 5636, 5580, 5538, 5596, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160088064 dispatch_meta_bytes=229632 combine_payload_bytes=160088064 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160317696 ingress_bytes=235585536 link_bytes=235585536 token_bytes=320405760 weight_movement_bytes=75497472 total_nvlink_bytes=395903232 weight_to_token_ratio=0.235631 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792533725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38794 dup_rows=21026 dup_rows_remote=18214 dedup_ratio=0.320831 router_tokens_per_expert=[544, 505, 507, 525, 518, 523, 516, 478, 495, 478, 520, 502, 532, 518, 528, 498, 514, 559, 523, 513, 541, 544, 471, 508, 540, 466, 481, 493, 484, 491, 505, 513, 495, 532, 493, 529, 478, 525, 534, 528, 524, 522, 517, 533, 524, 529, 478, 480, 536, 515, 540, 494, 481, 505, 499, 509, 533, 518, 480, 544, 552, 528, 476, 504] payload_rows_per_dst=[5716, 5714, 5706, 5380, 5494, 5600, 5410, 5490] dup_rows_per_dst=[2812, 2862, 2852, 2294, 2498, 2740, 2428, 2540] combine_rows_per_src=[5716, 5714, 5706, 5380, 5494, 5600, 5410, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=228800 combine_payload_bytes=159006720 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159235520 ingress_bytes=209338368 link_bytes=209338368 token_bytes=318242240 weight_movement_bytes=50331648 total_nvlink_bytes=368573888 weight_to_token_ratio=0.158155 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592823103e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159940608 dispatch_meta_bytes=230056 combine_payload_bytes=159940608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160170664 ingress_bytes=185106432 link_bytes=185106432 token_bytes=320111272 weight_movement_bytes=25165824 total_nvlink_bytes=345277096 weight_to_token_ratio=0.078616 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408446069e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159473664 dispatch_meta_bytes=229080 combine_payload_bytes=159473664 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159702744 ingress_bytes=234971136 link_bytes=234971136 token_bytes=319176408 weight_movement_bytes=75497472 total_nvlink_bytes=394673880 weight_to_token_ratio=0.236538 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787858851e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229816 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159900088 ingress_bytes=184836096 link_bytes=184836096 token_bytes=319570360 weight_movement_bytes=25165824 total_nvlink_bytes=344736184 weight_to_token_ratio=0.078749 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406389124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38924 dup_rows=21026 dup_rows_remote=18420 dedup_ratio=0.320831 router_tokens_per_expert=[518, 512, 470, 537, 526, 527, 504, 487, 490, 515, 480, 496, 519, 491, 517, 516, 503, 533, 523, 515, 507, 515, 512, 488, 505, 532, 494, 506, 529, 502, 469, 522, 479, 540, 535, 509, 505, 502, 577, 577, 520, 529, 504, 545, 535, 474, 531, 498, 539, 474, 515, 470, 497, 513, 525, 524, 539, 532, 535, 488, 524, 479, 523, 471] payload_rows_per_dst=[5578, 5494, 5586, 5564, 5632, 5608, 5494, 5554] dup_rows_per_dst=[2584, 2554, 2606, 2554, 2816, 2664, 2620, 2628] combine_rows_per_src=[5578, 5494, 5586, 5564, 5632, 5608, 5494, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158900224 dispatch_meta_bytes=228032 combine_payload_bytes=158900224 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159128256 ingress_bytes=184066048 link_bytes=184066048 token_bytes=318028480 weight_movement_bytes=25165824 total_nvlink_bytes=343194304 weight_to_token_ratio=0.079131 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400529949e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=39066 dup_rows=20916 dup_rows_remote=18368 dedup_ratio=0.319153 router_tokens_per_expert=[514, 507, 506, 501, 564, 520, 554, 526, 536, 495, 483, 536, 535, 507, 508, 548, 485, 537, 462, 511, 504, 532, 490, 534, 534, 527, 456, 509, 522, 464, 512, 527, 506, 488, 506, 517, 527, 506, 510, 494, 511, 514, 528, 505, 520, 504, 546, 533, 468, 501, 527, 516, 506, 492, 512, 505, 492, 515, 510, 503, 542, 524, 515, 479] payload_rows_per_dst=[5664, 5624, 5568, 5554, 5502, 5612, 5474, 5622] dup_rows_per_dst=[2720, 2672, 2542, 2548, 2606, 2710, 2580, 2538] combine_rows_per_src=[5664, 5624, 5568, 5554, 5502, 5612, 5474, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44398 payload_rows_remote=38818 dup_rows=21138 dup_rows_remote=18484 dedup_ratio=0.322540 router_tokens_per_expert=[514, 516, 489, 551, 514, 485, 498, 528, 513, 509, 513, 483, 545, 527, 500, 501, 485, 528, 535, 498, 558, 476, 539, 481, 514, 504, 485, 519, 524, 523, 524, 506, 509, 542, 468, 491, 526, 542, 551, 488, 529, 473, 502, 514, 514, 545, 501, 497, 531, 469, 507, 513, 526, 554, 554, 515, 477, 517, 512, 517, 507, 498, 510, 484] payload_rows_per_dst=[5532, 5546, 5600, 5572, 5580, 5548, 5572, 5448] dup_rows_per_dst=[2658, 2636, 2600, 2626, 2654, 2602, 2766, 2596] combine_rows_per_src=[5532, 5546, 5600, 5572, 5580, 5548, 5572, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=229376 combine_payload_bytes=159432704 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159662080 ingress_bytes=234930176 link_bytes=234930176 token_bytes=319094784 weight_movement_bytes=75497472 total_nvlink_bytes=394592256 weight_to_token_ratio=0.236599 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787547193e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38926 dup_rows=21072 dup_rows_remote=18372 dedup_ratio=0.321533 router_tokens_per_expert=[538, 480, 513, 508, 486, 509, 525, 494, 511, 525, 529, 516, 493, 507, 511, 532, 490, 523, 491, 508, 521, 530, 531, 529, 540, 534, 519, 501, 492, 551, 475, 495, 510, 529, 506, 515, 517, 517, 442, 552, 523, 486, 486, 503, 491, 480, 503, 557, 518, 515, 518, 513, 526, 510, 516, 503, 494, 549, 501, 505, 530, 506, 488, 552] payload_rows_per_dst=[5544, 5488, 5590, 5602, 5592, 5506, 5538, 5604] dup_rows_per_dst=[2562, 2760, 2656, 2612, 2584, 2552, 2700, 2646] combine_rows_per_src=[5544, 5488, 5590, 5602, 5592, 5506, 5538, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44826 payload_rows_remote=39232 dup_rows=20710 dup_rows_remote=18178 dedup_ratio=0.316010 router_tokens_per_expert=[474, 521, 523, 521, 534, 509, 516, 521, 493, 530, 484, 500, 520, 507, 496, 515, 514, 523, 519, 546, 492, 488, 487, 536, 515, 507, 501, 478, 519, 516, 491, 529, 518, 514, 512, 500, 496, 533, 560, 486, 483, 514, 514, 486, 496, 550, 512, 508, 503, 509, 502, 525, 528, 497, 499, 508, 523, 509, 536, 498, 529, 511, 528, 556] payload_rows_per_dst=[5598, 5528, 5578, 5546, 5702, 5594, 5554, 5726] dup_rows_per_dst=[2640, 2562, 2632, 2566, 2536, 2532, 2588, 2654] combine_rows_per_src=[5598, 5528, 5578, 5546, 5702, 5594, 5554, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44482 payload_rows_remote=38890 dup_rows=21054 dup_rows_remote=18478 dedup_ratio=0.321259 router_tokens_per_expert=[529, 484, 498, 512, 540, 493, 534, 500, 483, 496, 552, 494, 518, 538, 495, 501, 514, 523, 484, 496, 514, 540, 511, 506, 507, 513, 549, 517, 502, 486, 503, 513, 507, 507, 517, 552, 536, 502, 515, 514, 518, 502, 512, 516, 505, 489, 542, 508, 510, 485, 521, 515, 562, 513, 532, 537, 475, 467, 506, 499, 509, 520, 512, 518] payload_rows_per_dst=[5550, 5540, 5472, 5638, 5518, 5586, 5586, 5592] dup_rows_per_dst=[2630, 2614, 2454, 2728, 2630, 2778, 2644, 2576] combine_rows_per_src=[5550, 5540, 5472, 5638, 5518, 5586, 5586, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=229736 combine_payload_bytes=160014336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160244072 ingress_bytes=185180160 link_bytes=185180160 token_bytes=320258408 weight_movement_bytes=25165824 total_nvlink_bytes=345424232 weight_to_token_ratio=0.078580 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409007054e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44498 payload_rows_remote=38982 dup_rows=21038 dup_rows_remote=18492 dedup_ratio=0.321014 router_tokens_per_expert=[496, 503, 542, 498, 525, 480, 537, 502, 482, 466, 545, 513, 520, 505, 489, 511, 520, 553, 524, 495, 541, 506, 474, 537, 494, 462, 505, 511, 501, 539, 517, 522, 519, 549, 500, 530, 486, 502, 489, 498, 490, 510, 520, 509, 507, 517, 514, 542, 531, 510, 522, 510, 530, 492, 501, 551, 532, 538, 519, 488, 507, 542, 510, 488] payload_rows_per_dst=[5540, 5516, 5626, 5486, 5552, 5584, 5604, 5590] dup_rows_per_dst=[2626, 2546, 2674, 2616, 2594, 2634, 2690, 2658] combine_rows_per_src=[5540, 5516, 5626, 5486, 5552, 5584, 5604, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158998528 dispatch_meta_bytes=229208 combine_payload_bytes=158998528 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159227736 ingress_bytes=234496000 link_bytes=234496000 token_bytes=318226264 weight_movement_bytes=75497472 total_nvlink_bytes=393723736 weight_to_token_ratio=0.237245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784243615e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44584 payload_rows_remote=39008 dup_rows=20952 dup_rows_remote=18418 dedup_ratio=0.319702 router_tokens_per_expert=[522, 513, 526, 496, 476, 515, 476, 484, 527, 517, 516, 490, 502, 533, 513, 525, 499, 514, 524, 537, 564, 488, 536, 514, 516, 544, 509, 550, 502, 520, 504, 517, 506, 492, 501, 497, 534, 528, 489, 519, 518, 473, 536, 516, 503, 475, 489, 503, 523, 553, 506, 545, 512, 511, 503, 515, 505, 526, 510, 490, 511, 514, 492, 504] payload_rows_per_dst=[5576, 5698, 5670, 5614, 5498, 5424, 5584, 5520] dup_rows_per_dst=[2534, 2758, 2682, 2710, 2540, 2602, 2542, 2584] combine_rows_per_src=[5576, 5698, 5670, 5614, 5498, 5424, 5584, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160694272 dispatch_meta_bytes=229640 combine_payload_bytes=160694272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160923912 ingress_bytes=185860096 link_bytes=185860096 token_bytes=321618184 weight_movement_bytes=25165824 total_nvlink_bytes=346784008 weight_to_token_ratio=0.078248 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.414180581e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159440896 dispatch_meta_bytes=229192 combine_payload_bytes=159440896 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159670088 ingress_bytes=234938368 link_bytes=234938368 token_bytes=319110984 weight_movement_bytes=75497472 total_nvlink_bytes=394608456 weight_to_token_ratio=0.236587 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787609524e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=229472 combine_payload_bytes=159293440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159522912 ingress_bytes=184459264 link_bytes=184459264 token_bytes=318816352 weight_movement_bytes=25165824 total_nvlink_bytes=343982176 weight_to_token_ratio=0.078935 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403521868e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229896 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159900168 ingress_bytes=184836096 link_bytes=184836096 token_bytes=319570440 weight_movement_bytes=25165824 total_nvlink_bytes=344736264 weight_to_token_ratio=0.078749 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406389124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44398 payload_rows_remote=38902 dup_rows=21138 dup_rows_remote=18552 dedup_ratio=0.322540 router_tokens_per_expert=[490, 524, 510, 543, 532, 486, 512, 496, 515, 514, 515, 535, 539, 531, 492, 534, 487, 557, 502, 524, 519, 438, 524, 490, 531, 528, 536, 541, 479, 501, 480, 522, 543, 502, 590, 517, 482, 496, 504, 490, 493, 497, 493, 500, 529, 496, 528, 517, 484, 482, 498, 517, 544, 523, 502, 519, 499, 526, 514, 494, 501, 520, 531, 510] payload_rows_per_dst=[5492, 5646, 5496, 5574, 5546, 5494, 5584, 5566] dup_rows_per_dst=[2694, 2704, 2586, 2662, 2702, 2612, 2554, 2624] combine_rows_per_src=[5492, 5646, 5496, 5574, 5546, 5494, 5584, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159776768 dispatch_meta_bytes=229704 combine_payload_bytes=159776768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160006472 ingress_bytes=184942592 link_bytes=184942592 token_bytes=319783240 weight_movement_bytes=25165824 total_nvlink_bytes=344949064 weight_to_token_ratio=0.078697 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407199436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44470 payload_rows_remote=38998 dup_rows=21066 dup_rows_remote=18450 dedup_ratio=0.321442 router_tokens_per_expert=[561, 516, 497, 495, 536, 505, 504, 517, 506, 488, 540, 509, 522, 526, 523, 493, 503, 513, 509, 521, 518, 473, 555, 487, 500, 512, 509, 504, 530, 473, 509, 507, 493, 505, 517, 460, 544, 525, 503, 506, 537, 513, 531, 527, 495, 513, 497, 515, 542, 486, 491, 536, 517, 478, 525, 509, 486, 506, 525, 526, 473, 551, 525, 550] payload_rows_per_dst=[5580, 5592, 5540, 5472, 5560, 5618, 5568, 5540] dup_rows_per_dst=[2682, 2622, 2618, 2616, 2546, 2638, 2600, 2744] combine_rows_per_src=[5580, 5592, 5540, 5472, 5560, 5618, 5568, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=229816 combine_payload_bytes=159342592 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159572408 ingress_bytes=209674240 link_bytes=209674240 token_bytes=318915000 weight_movement_bytes=50331648 total_nvlink_bytes=369246648 weight_to_token_ratio=0.157822 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595378701e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38854 dup_rows=21098 dup_rows_remote=18460 dedup_ratio=0.321930 router_tokens_per_expert=[538, 477, 517, 484, 528, 504, 522, 534, 501, 507, 523, 522, 512, 534, 459, 496, 500, 497, 533, 524, 485, 525, 486, 529, 537, 461, 502, 575, 538, 535, 527, 504, 556, 497, 542, 499, 494, 495, 526, 502, 494, 476, 533, 500, 528, 486, 511, 502, 521, 506, 495, 502, 496, 514, 506, 493, 538, 516, 535, 483, 526, 551, 545, 484] payload_rows_per_dst=[5586, 5552, 5496, 5596, 5584, 5528, 5494, 5602] dup_rows_per_dst=[2622, 2556, 2662, 2762, 2638, 2532, 2572, 2754] combine_rows_per_src=[5586, 5552, 5496, 5596, 5584, 5528, 5494, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44352 payload_rows_remote=38860 dup_rows=21184 dup_rows_remote=18610 dedup_ratio=0.323242 router_tokens_per_expert=[518, 526, 515, 523, 517, 497, 523, 513, 511, 529, 507, 530, 496, 504, 519, 504, 487, 509, 501, 501, 541, 515, 508, 515, 501, 514, 541, 538, 517, 516, 539, 512, 501, 521, 473, 514, 472, 516, 506, 538, 529, 508, 496, 494, 489, 539, 452, 526, 509, 502, 478, 511, 543, 513, 532, 495, 525, 501, 505, 498, 521, 553, 520, 501] payload_rows_per_dst=[5562, 5632, 5498, 5554, 5510, 5492, 5592, 5512] dup_rows_per_dst=[2702, 2568, 2656, 2802, 2572, 2574, 2574, 2736] combine_rows_per_src=[5562, 5632, 5498, 5554, 5510, 5492, 5592, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44342 payload_rows_remote=38772 dup_rows=21194 dup_rows_remote=18484 dedup_ratio=0.323395 router_tokens_per_expert=[487, 521, 520, 528, 515, 531, 486, 514, 502, 489, 521, 475, 543, 501, 500, 528, 526, 524, 495, 515, 521, 507, 530, 469, 489, 477, 485, 515, 550, 551, 514, 542, 481, 492, 514, 498, 522, 520, 534, 499, 519, 531, 475, 506, 490, 527, 490, 532, 490, 509, 512, 523, 507, 516, 566, 517, 459, 538, 531, 511, 510, 541, 528, 509] payload_rows_per_dst=[5546, 5526, 5556, 5538, 5532, 5524, 5570, 5550] dup_rows_per_dst=[2658, 2592, 2618, 2708, 2588, 2616, 2710, 2704] combine_rows_per_src=[5546, 5526, 5556, 5538, 5532, 5524, 5570, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44366 payload_rows_remote=38546 dup_rows=21170 dup_rows_remote=18310 dedup_ratio=0.323029 router_tokens_per_expert=[517, 534, 498, 499, 522, 504, 541, 505, 505, 513, 504, 491, 512, 524, 492, 519, 490, 526, 503, 502, 539, 496, 527, 538, 515, 497, 500, 542, 500, 472, 547, 516, 502, 551, 499, 509, 519, 508, 525, 522, 481, 511, 479, 492, 494, 473, 512, 490, 555, 512, 518, 556, 462, 510, 534, 513, 522, 530, 538, 525, 509, 514, 506, 507] payload_rows_per_dst=[5324, 5488, 5634, 5418, 5680, 5402, 5600, 5820] dup_rows_per_dst=[2452, 2632, 2608, 2504, 2932, 2462, 2720, 2860] combine_rows_per_src=[5324, 5488, 5634, 5418, 5680, 5402, 5600, 5820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38908 dup_rows=21002 dup_rows_remote=18324 dedup_ratio=0.320465 router_tokens_per_expert=[557, 521, 530, 511, 525, 495, 540, 467, 506, 525, 537, 501, 529, 535, 498, 521, 475, 531, 525, 508, 509, 560, 520, 494, 460, 536, 531, 529, 505, 494, 501, 532, 526, 527, 500, 526, 564, 509, 470, 479, 529, 505, 489, 473, 497, 493, 481, 543, 495, 521, 527, 513, 501, 489, 498, 533, 532, 534, 471, 528, 473, 492, 552, 490] payload_rows_per_dst=[5582, 5626, 5574, 5506, 5566, 5484, 5568, 5628] dup_rows_per_dst=[2710, 2678, 2670, 2670, 2636, 2536, 2586, 2516] combine_rows_per_src=[5582, 5626, 5574, 5506, 5566, 5484, 5568, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159735808 dispatch_meta_bytes=229792 combine_payload_bytes=159735808 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159965600 ingress_bytes=210067456 link_bytes=210067456 token_bytes=319701408 weight_movement_bytes=50331648 total_nvlink_bytes=370033056 weight_to_token_ratio=0.157433 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598370621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=38854 dup_rows=21070 dup_rows_remote=18338 dedup_ratio=0.321503 router_tokens_per_expert=[494, 516, 508, 505, 497, 548, 582, 502, 512, 493, 489, 530, 497, 545, 493, 487, 525, 553, 515, 475, 535, 503, 556, 513, 507, 513, 500, 480, 533, 535, 505, 504, 483, 523, 469, 507, 461, 516, 498, 515, 517, 492, 526, 512, 505, 525, 494, 527, 517, 512, 516, 489, 544, 527, 479, 560, 507, 503, 505, 532, 543, 478, 510, 526] payload_rows_per_dst=[5612, 5550, 5608, 5542, 5428, 5850, 5604, 5272] dup_rows_per_dst=[2732, 2572, 2716, 2612, 2516, 3174, 2640, 2108] combine_rows_per_src=[5612, 5550, 5608, 5542, 5428, 5850, 5604, 5272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159145984 dispatch_meta_bytes=229256 combine_payload_bytes=159145984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159375240 ingress_bytes=184311808 link_bytes=184311808 token_bytes=318521224 weight_movement_bytes=25165824 total_nvlink_bytes=343687048 weight_to_token_ratio=0.079008 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402399898e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38906 dup_rows=21118 dup_rows_remote=18554 dedup_ratio=0.322235 router_tokens_per_expert=[523, 470, 511, 538, 556, 507, 541, 510, 463, 504, 499, 534, 523, 523, 494, 487, 526, 486, 501, 533, 486, 504, 474, 528, 536, 516, 498, 551, 521, 508, 515, 486, 505, 502, 505, 495, 494, 520, 462, 529, 505, 535, 535, 516, 533, 512, 533, 515, 517, 494, 553, 503, 506, 527, 524, 520, 493, 501, 506, 472, 528, 514, 535, 527] payload_rows_per_dst=[5582, 5546, 5512, 5624, 5462, 5558, 5552, 5582] dup_rows_per_dst=[2730, 2508, 2564, 2638, 2562, 2810, 2736, 2570] combine_rows_per_src=[5582, 5546, 5512, 5624, 5462, 5558, 5552, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159170560 dispatch_meta_bytes=229880 combine_payload_bytes=159170560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159400440 ingress_bytes=184336384 link_bytes=184336384 token_bytes=318571000 weight_movement_bytes=25165824 total_nvlink_bytes=343736824 weight_to_token_ratio=0.078996 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402586893e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=157884416 dispatch_meta_bytes=227424 combine_payload_bytes=157884416 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158111840 ingress_bytes=208216064 link_bytes=208216064 token_bytes=315996256 weight_movement_bytes=50331648 total_nvlink_bytes=366327904 weight_to_token_ratio=0.159279 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.584283667e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158810112 dispatch_meta_bytes=229024 combine_payload_bytes=158810112 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159039136 ingress_bytes=209141760 link_bytes=209141760 token_bytes=317849248 weight_movement_bytes=50331648 total_nvlink_bytes=368180896 weight_to_token_ratio=0.158351 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591327144e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159367168 dispatch_meta_bytes=228928 combine_payload_bytes=159367168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159596096 ingress_bytes=184532992 link_bytes=184532992 token_bytes=318963264 weight_movement_bytes=25165824 total_nvlink_bytes=344129088 weight_to_token_ratio=0.078899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404082853e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159145984 dispatch_meta_bytes=228768 combine_payload_bytes=159145984 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159374752 ingress_bytes=234643456 link_bytes=234643456 token_bytes=318520736 weight_movement_bytes=75497472 total_nvlink_bytes=394018208 weight_to_token_ratio=0.237025 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785365585e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229840 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159588816 ingress_bytes=260022272 link_bytes=260022272 token_bytes=318947792 weight_movement_bytes=100663296 total_nvlink_bytes=419611088 weight_to_token_ratio=0.315611 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978469051e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38884 dup_rows=21088 dup_rows_remote=18384 dedup_ratio=0.321777 router_tokens_per_expert=[524, 497, 519, 525, 505, 516, 543, 514, 539, 495, 503, 540, 517, 484, 542, 533, 508, 554, 518, 511, 537, 529, 527, 516, 542, 518, 530, 499, 547, 516, 473, 509, 517, 497, 511, 515, 491, 482, 523, 486, 534, 517, 512, 527, 515, 479, 463, 500, 482, 492, 548, 477, 506, 494, 483, 506, 530, 493, 522, 484, 534, 502, 523, 493] payload_rows_per_dst=[5626, 5590, 5632, 5564, 5532, 5450, 5508, 5546] dup_rows_per_dst=[2660, 2716, 2768, 2704, 2512, 2644, 2468, 2616] combine_rows_per_src=[5626, 5590, 5632, 5564, 5532, 5450, 5508, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44348 payload_rows_remote=38760 dup_rows=21188 dup_rows_remote=18648 dedup_ratio=0.323303 router_tokens_per_expert=[495, 520, 507, 460, 481, 511, 531, 534, 570, 482, 496, 536, 546, 503, 534, 530, 493, 466, 519, 514, 508, 546, 536, 507, 527, 499, 502, 532, 502, 499, 500, 486, 489, 487, 530, 489, 539, 483, 522, 525, 487, 532, 508, 530, 539, 508, 504, 528, 511, 535, 514, 519, 496, 508, 515, 501, 531, 521, 515, 500, 502, 510, 497, 521] payload_rows_per_dst=[5514, 5580, 5546, 5534, 5588, 5528, 5580, 5478] dup_rows_per_dst=[2564, 2814, 2632, 2560, 2540, 2744, 2618, 2716] combine_rows_per_src=[5514, 5580, 5546, 5534, 5588, 5528, 5580, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=38964 dup_rows=20990 dup_rows_remote=18262 dedup_ratio=0.320282 router_tokens_per_expert=[502, 509, 539, 484, 504, 523, 490, 543, 512, 477, 537, 510, 485, 501, 487, 523, 529, 510, 511, 538, 509, 527, 515, 532, 489, 520, 516, 500, 507, 551, 504, 490, 509, 505, 509, 493, 461, 530, 483, 513, 512, 522, 538, 501, 488, 509, 561, 524, 536, 528, 514, 517, 539, 472, 540, 485, 506, 543, 507, 499, 529, 503, 511, 507] payload_rows_per_dst=[5584, 5520, 5670, 5534, 5452, 5582, 5592, 5612] dup_rows_per_dst=[2604, 2544, 2672, 2620, 2554, 2728, 2670, 2598] combine_rows_per_src=[5584, 5520, 5670, 5534, 5452, 5582, 5592, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=229072 combine_payload_bytes=159268864 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159497936 ingress_bytes=184434688 link_bytes=184434688 token_bytes=318766800 weight_movement_bytes=25165824 total_nvlink_bytes=343932624 weight_to_token_ratio=0.078947 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403334873e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44808 payload_rows_remote=39124 dup_rows=20728 dup_rows_remote=18042 dedup_ratio=0.316284 router_tokens_per_expert=[524, 512, 499, 501, 479, 526, 499, 511, 486, 479, 492, 513, 539, 517, 508, 519, 519, 519, 555, 504, 519, 552, 557, 527, 515, 514, 512, 512, 543, 510, 515, 503, 504, 520, 524, 549, 497, 542, 506, 498, 518, 535, 494, 516, 485, 482, 507, 527, 470, 562, 512, 511, 483, 514, 510, 514, 492, 503, 526, 470, 466, 485, 544, 522] payload_rows_per_dst=[5488, 5546, 5544, 5642, 5668, 5512, 5724, 5684] dup_rows_per_dst=[2442, 2560, 2416, 2606, 2612, 2616, 2790, 2686] combine_rows_per_src=[5488, 5546, 5544, 5642, 5668, 5512, 5724, 5684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=39086 dup_rows=21000 dup_rows_remote=18424 dedup_ratio=0.320435 router_tokens_per_expert=[545, 531, 538, 531, 497, 494, 498, 520, 527, 506, 489, 463, 497, 502, 519, 510, 543, 528, 511, 491, 496, 484, 525, 522, 522, 504, 519, 492, 513, 515, 475, 502, 513, 515, 512, 533, 545, 521, 510, 519, 481, 513, 503, 497, 521, 554, 553, 567, 511, 501, 506, 515, 503, 508, 502, 474, 533, 487, 499, 495, 505, 511, 546, 506] payload_rows_per_dst=[5664, 5450, 5532, 5544, 5624, 5616, 5522, 5584] dup_rows_per_dst=[2644, 2576, 2668, 2540, 2712, 2762, 2518, 2580] combine_rows_per_src=[5664, 5450, 5532, 5544, 5624, 5616, 5522, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38810 dup_rows=21082 dup_rows_remote=18392 dedup_ratio=0.321686 router_tokens_per_expert=[511, 508, 482, 543, 523, 524, 515, 534, 502, 503, 526, 515, 496, 496, 505, 506, 516, 533, 491, 507, 526, 523, 525, 520, 460, 519, 511, 488, 532, 502, 505, 522, 564, 524, 502, 535, 504, 504, 464, 525, 517, 504, 533, 502, 499, 562, 529, 453, 541, 496, 519, 485, 526, 509, 487, 512, 507, 488, 499, 543, 515, 517, 506, 528] payload_rows_per_dst=[5644, 5566, 5498, 5544, 5664, 5396, 5554, 5588] dup_rows_per_dst=[2690, 2828, 2488, 2584, 2910, 2368, 2596, 2618] combine_rows_per_src=[5644, 5566, 5498, 5544, 5664, 5396, 5554, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=229632 combine_payload_bytes=158760960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158990592 ingress_bytes=183926784 link_bytes=183926784 token_bytes=317751552 weight_movement_bytes=25165824 total_nvlink_bytes=342917376 weight_to_token_ratio=0.079200 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399470311e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44578 payload_rows_remote=39046 dup_rows=20958 dup_rows_remote=18382 dedup_ratio=0.319794 router_tokens_per_expert=[532, 519, 509, 498, 489, 520, 510, 473, 482, 509, 512, 550, 501, 521, 491, 530, 502, 546, 509, 518, 517, 539, 527, 539, 531, 528, 541, 495, 556, 521, 487, 490, 514, 529, 514, 527, 505, 492, 494, 497, 528, 524, 501, 496, 495, 530, 522, 515, 491, 510, 515, 486, 523, 512, 509, 508, 512, 499, 482, 520, 518, 491, 505, 512] payload_rows_per_dst=[5516, 5574, 5676, 5582, 5594, 5562, 5532, 5542] dup_rows_per_dst=[2584, 2618, 2718, 2716, 2550, 2660, 2576, 2536] combine_rows_per_src=[5516, 5574, 5676, 5582, 5594, 5562, 5532, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=39066 dup_rows=20916 dup_rows_remote=18250 dedup_ratio=0.319153 router_tokens_per_expert=[520, 505, 536, 516, 486, 535, 522, 483, 536, 519, 502, 496, 486, 511, 526, 497, 546, 453, 526, 528, 542, 497, 526, 492, 526, 516, 471, 492, 526, 551, 497, 519, 482, 510, 482, 527, 518, 553, 521, 543, 480, 529, 492, 511, 532, 522, 502, 513, 532, 500, 495, 487, 502, 502, 552, 516, 531, 500, 539, 518, 484, 486, 512, 511] payload_rows_per_dst=[5622, 5586, 5554, 5596, 5624, 5500, 5564, 5574] dup_rows_per_dst=[2584, 2560, 2666, 2600, 2648, 2662, 2608, 2588] combine_rows_per_src=[5622, 5586, 5554, 5596, 5624, 5500, 5564, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=228904 combine_payload_bytes=159596544 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159825448 ingress_bytes=235094016 link_bytes=235094016 token_bytes=319421992 weight_movement_bytes=75497472 total_nvlink_bytes=394919464 weight_to_token_ratio=0.236357 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788793826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160251904 dispatch_meta_bytes=228664 combine_payload_bytes=160251904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160480568 ingress_bytes=185417728 link_bytes=185417728 token_bytes=320732472 weight_movement_bytes=25165824 total_nvlink_bytes=345898296 weight_to_token_ratio=0.078464 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410814672e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160096256 dispatch_meta_bytes=230040 combine_payload_bytes=160096256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160326296 ingress_bytes=185262080 link_bytes=185262080 token_bytes=320422552 weight_movement_bytes=25165824 total_nvlink_bytes=345588376 weight_to_token_ratio=0.078539 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409630370e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158965760 dispatch_meta_bytes=228808 combine_payload_bytes=158965760 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159194568 ingress_bytes=209297408 link_bytes=209297408 token_bytes=318160328 weight_movement_bytes=50331648 total_nvlink_bytes=368491976 weight_to_token_ratio=0.158196 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592511445e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159932416 dispatch_meta_bytes=229712 combine_payload_bytes=159932416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160162128 ingress_bytes=185098240 link_bytes=185098240 token_bytes=320094544 weight_movement_bytes=25165824 total_nvlink_bytes=345260368 weight_to_token_ratio=0.078620 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408383737e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=229264 combine_payload_bytes=160014336 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160243600 ingress_bytes=210345984 link_bytes=210345984 token_bytes=320257936 weight_movement_bytes=50331648 total_nvlink_bytes=370589584 weight_to_token_ratio=0.157160 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600489897e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44354 payload_rows_remote=38920 dup_rows=21182 dup_rows_remote=18596 dedup_ratio=0.323212 router_tokens_per_expert=[512, 504, 528, 556, 502, 496, 512, 519, 530, 534, 490, 509, 502, 477, 484, 527, 523, 511, 515, 521, 528, 532, 516, 521, 514, 453, 548, 492, 453, 499, 503, 548, 558, 516, 515, 498, 526, 490, 515, 503, 500, 543, 538, 532, 542, 517, 524, 520, 545, 496, 531, 474, 513, 508, 492, 530, 511, 516, 486, 485, 511, 511, 489, 474] payload_rows_per_dst=[5594, 5506, 5652, 5434, 5550, 5598, 5510, 5510] dup_rows_per_dst=[2664, 2600, 2682, 2586, 2692, 2834, 2668, 2456] combine_rows_per_src=[5594, 5506, 5652, 5434, 5550, 5598, 5510, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38892 dup_rows=21072 dup_rows_remote=18388 dedup_ratio=0.321533 router_tokens_per_expert=[541, 513, 487, 493, 507, 518, 534, 527, 516, 501, 504, 483, 506, 515, 508, 516, 482, 500, 512, 549, 543, 542, 506, 529, 475, 487, 556, 531, 521, 496, 509, 516, 540, 524, 499, 461, 476, 546, 543, 539, 456, 549, 512, 505, 542, 485, 532, 489, 508, 516, 510, 502, 556, 514, 483, 503, 524, 511, 498, 519, 508, 471, 515, 509] payload_rows_per_dst=[5572, 5502, 5632, 5602, 5572, 5502, 5588, 5494] dup_rows_per_dst=[2668, 2596, 2694, 2580, 2684, 2638, 2596, 2616] combine_rows_per_src=[5572, 5502, 5632, 5602, 5572, 5502, 5588, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38966 dup_rows=21142 dup_rows_remote=18474 dedup_ratio=0.322601 router_tokens_per_expert=[493, 520, 459, 491, 514, 487, 531, 532, 523, 546, 547, 460, 489, 514, 534, 561, 551, 533, 513, 530, 486, 504, 510, 516, 521, 533, 536, 536, 489, 491, 504, 516, 508, 518, 498, 518, 497, 489, 528, 524, 498, 487, 494, 517, 499, 523, 522, 508, 498, 489, 460, 516, 492, 514, 514, 488, 566, 519, 508, 506, 524, 555, 506, 515] payload_rows_per_dst=[5506, 5662, 5592, 5574, 5588, 5428, 5424, 5620] dup_rows_per_dst=[2548, 2686, 2694, 2678, 2572, 2668, 2518, 2778] combine_rows_per_src=[5506, 5662, 5592, 5574, 5588, 5428, 5424, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=230064 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159646384 ingress_bytes=184582144 link_bytes=184582144 token_bytes=319062704 weight_movement_bytes=25165824 total_nvlink_bytes=344228528 weight_to_token_ratio=0.078874 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404456843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44446 payload_rows_remote=38894 dup_rows=21090 dup_rows_remote=18386 dedup_ratio=0.321808 router_tokens_per_expert=[518, 516, 525, 525, 475, 536, 519, 481, 516, 491, 552, 507, 478, 525, 527, 532, 500, 519, 520, 504, 505, 540, 508, 547, 516, 511, 526, 511, 507, 504, 490, 520, 471, 483, 544, 515, 502, 472, 531, 491, 503, 534, 509, 526, 489, 499, 545, 511, 546, 526, 515, 498, 485, 492, 509, 512, 530, 502, 480, 515, 488, 522, 536, 536] payload_rows_per_dst=[5516, 5530, 5462, 5536, 5640, 5638, 5572, 5552] dup_rows_per_dst=[2674, 2726, 2338, 2634, 2864, 2594, 2556, 2704] combine_rows_per_src=[5516, 5530, 5462, 5536, 5640, 5638, 5572, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159301632 dispatch_meta_bytes=229120 combine_payload_bytes=159301632 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159530752 ingress_bytes=184467456 link_bytes=184467456 token_bytes=318832384 weight_movement_bytes=25165824 total_nvlink_bytes=343998208 weight_to_token_ratio=0.078931 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403584200e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=39046 dup_rows=20988 dup_rows_remote=18246 dedup_ratio=0.320251 router_tokens_per_expert=[499, 499, 553, 505, 560, 479, 495, 521, 523, 549, 505, 523, 520, 534, 486, 482, 479, 498, 509, 505, 525, 519, 545, 510, 503, 517, 501, 511, 506, 533, 537, 502, 476, 496, 489, 546, 471, 465, 504, 514, 473, 472, 525, 548, 470, 552, 483, 511, 506, 487, 521, 498, 539, 556, 524, 515, 504, 493, 538, 522, 525, 522, 537, 553] payload_rows_per_dst=[5630, 5502, 5514, 5540, 5480, 5574, 5632, 5676] dup_rows_per_dst=[2592, 2742, 2666, 2680, 2442, 2494, 2660, 2712] combine_rows_per_src=[5630, 5502, 5514, 5540, 5480, 5574, 5632, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44342 payload_rows_remote=38802 dup_rows=21194 dup_rows_remote=18460 dedup_ratio=0.323395 router_tokens_per_expert=[467, 467, 535, 476, 536, 489, 511, 498, 513, 518, 537, 513, 506, 527, 524, 529, 527, 539, 539, 505, 503, 498, 480, 521, 480, 500, 498, 477, 505, 517, 509, 492, 489, 540, 495, 507, 513, 500, 482, 515, 492, 525, 512, 534, 527, 527, 515, 503, 520, 557, 508, 518, 525, 534, 491, 571, 530, 508, 508, 525, 517, 525, 493, 526] payload_rows_per_dst=[5540, 5452, 5590, 5590, 5526, 5560, 5590, 5494] dup_rows_per_dst=[2734, 2606, 2664, 2710, 2556, 2794, 2704, 2426] combine_rows_per_src=[5540, 5452, 5590, 5590, 5526, 5560, 5590, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44474 payload_rows_remote=38882 dup_rows=21062 dup_rows_remote=18406 dedup_ratio=0.321381 router_tokens_per_expert=[503, 535, 532, 540, 489, 542, 488, 551, 539, 504, 546, 475, 474, 520, 549, 493, 538, 463, 515, 536, 513, 532, 509, 518, 487, 477, 497, 530, 488, 547, 510, 514, 526, 514, 472, 522, 514, 513, 517, 495, 496, 508, 553, 457, 522, 520, 519, 498, 526, 536, 551, 487, 484, 462, 505, 541, 533, 533, 461, 506, 517, 519, 504, 503] payload_rows_per_dst=[5564, 5584, 5592, 5576, 5476, 5504, 5570, 5608] dup_rows_per_dst=[2796, 2616, 2656, 2524, 2670, 2642, 2614, 2544] combine_rows_per_src=[5564, 5584, 5592, 5576, 5476, 5504, 5570, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44844 payload_rows_remote=39176 dup_rows=20692 dup_rows_remote=18014 dedup_ratio=0.315735 router_tokens_per_expert=[516, 503, 502, 529, 486, 499, 520, 491, 520, 499, 514, 526, 535, 518, 523, 505, 491, 498, 528, 528, 516, 519, 520, 468, 501, 509, 540, 487, 480, 546, 511, 524, 523, 536, 515, 459, 517, 490, 496, 513, 517, 428, 496, 506, 509, 545, 496, 529, 517, 500, 539, 502, 513, 529, 523, 550, 556, 516, 505, 504, 541, 504, 520, 522] payload_rows_per_dst=[5550, 5656, 5578, 5610, 5498, 5578, 5668, 5706] dup_rows_per_dst=[2542, 2624, 2558, 2586, 2600, 2474, 2678, 2630] combine_rows_per_src=[5550, 5656, 5578, 5610, 5498, 5578, 5668, 5706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159604736 dispatch_meta_bytes=229760 combine_payload_bytes=159604736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159834496 ingress_bytes=184770560 link_bytes=184770560 token_bytes=319439232 weight_movement_bytes=25165824 total_nvlink_bytes=344605056 weight_to_token_ratio=0.078781 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405890471e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=229120 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159538944 ingress_bytes=209641472 link_bytes=209641472 token_bytes=318848768 weight_movement_bytes=50331648 total_nvlink_bytes=369180416 weight_to_token_ratio=0.157854 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595129375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159932416 dispatch_meta_bytes=229168 combine_payload_bytes=159932416 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160161584 ingress_bytes=210264064 link_bytes=210264064 token_bytes=320094000 weight_movement_bytes=50331648 total_nvlink_bytes=370425648 weight_to_token_ratio=0.157240 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599866580e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158932992 dispatch_meta_bytes=229048 combine_payload_bytes=158932992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159162040 ingress_bytes=184098816 link_bytes=184098816 token_bytes=318095032 weight_movement_bytes=25165824 total_nvlink_bytes=343260856 weight_to_token_ratio=0.079114 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400779275e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44676 payload_rows_remote=39070 dup_rows=20860 dup_rows_remote=18154 dedup_ratio=0.318298 router_tokens_per_expert=[547, 542, 478, 506, 537, 537, 505, 540, 506, 556, 527, 512, 522, 500, 504, 491, 541, 500, 528, 498, 494, 496, 494, 535, 538, 546, 509, 507, 555, 489, 495, 517, 466, 489, 493, 534, 515, 526, 525, 486, 535, 513, 521, 467, 528, 517, 505, 480, 531, 543, 509, 514, 514, 516, 500, 509, 493, 530, 467, 493, 567, 493, 447, 490] payload_rows_per_dst=[5700, 5602, 5642, 5606, 5512, 5532, 5592, 5490] dup_rows_per_dst=[2684, 2634, 2530, 2706, 2556, 2600, 2680, 2470] combine_rows_per_src=[5700, 5602, 5642, 5606, 5512, 5532, 5592, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160464896 dispatch_meta_bytes=228760 combine_payload_bytes=160464896 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=160693656 ingress_bytes=261128192 link_bytes=261128192 token_bytes=321158552 weight_movement_bytes=100663296 total_nvlink_bytes=421821848 weight_to_token_ratio=0.313438 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.986883824e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159260672 dispatch_meta_bytes=229152 combine_payload_bytes=159260672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159489824 ingress_bytes=184426496 link_bytes=184426496 token_bytes=318750496 weight_movement_bytes=25165824 total_nvlink_bytes=343916320 weight_to_token_ratio=0.078951 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403272542e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44698 payload_rows_remote=39104 dup_rows=20838 dup_rows_remote=18152 dedup_ratio=0.317963 router_tokens_per_expert=[488, 507, 498, 488, 501, 546, 536, 525, 490, 508, 490, 522, 545, 472, 538, 502, 551, 531, 495, 558, 522, 446, 484, 559, 500, 494, 519, 521, 478, 488, 472, 472, 486, 536, 496, 523, 520, 533, 502, 544, 538, 502, 534, 496, 545, 504, 513, 518, 512, 539, 517, 551, 512, 524, 519, 503, 518, 503, 490, 469, 544, 481, 540, 510] payload_rows_per_dst=[5574, 5550, 5604, 5426, 5594, 5622, 5680, 5648] dup_rows_per_dst=[2604, 2584, 2688, 2462, 2686, 2678, 2674, 2462] combine_rows_per_src=[5574, 5550, 5604, 5426, 5594, 5622, 5680, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160030720 dispatch_meta_bytes=228896 combine_payload_bytes=160030720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160259616 ingress_bytes=185196544 link_bytes=185196544 token_bytes=320290336 weight_movement_bytes=25165824 total_nvlink_bytes=345456160 weight_to_token_ratio=0.078572 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409131717e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44424 payload_rows_remote=38788 dup_rows=21112 dup_rows_remote=18494 dedup_ratio=0.322144 router_tokens_per_expert=[526, 499, 496, 485, 538, 494, 544, 490, 506, 510, 544, 519, 473, 461, 509, 541, 462, 493, 541, 488, 520, 569, 521, 492, 509, 473, 537, 561, 504, 475, 506, 543, 550, 514, 525, 526, 476, 524, 549, 496, 515, 522, 522, 511, 517, 511, 503, 526, 525, 486, 524, 504, 511, 503, 540, 497, 536, 488, 481, 492, 479, 545, 513, 528] payload_rows_per_dst=[5564, 5528, 5506, 5550, 5592, 5636, 5490, 5558] dup_rows_per_dst=[2580, 2598, 2666, 2666, 2728, 2618, 2690, 2566] combine_rows_per_src=[5564, 5528, 5506, 5550, 5592, 5636, 5490, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38902 dup_rows=21034 dup_rows_remote=18316 dedup_ratio=0.320953 router_tokens_per_expert=[515, 511, 498, 482, 506, 470, 493, 543, 528, 543, 511, 511, 529, 505, 511, 533, 498, 515, 532, 473, 517, 519, 515, 523, 498, 511, 518, 466, 512, 516, 510, 549, 479, 542, 492, 495, 555, 491, 504, 528, 524, 552, 563, 532, 478, 501, 523, 498, 483, 501, 530, 502, 522, 474, 470, 509, 502, 525, 494, 533, 510, 532, 533, 530] payload_rows_per_dst=[5486, 5410, 5598, 5814, 5484, 5670, 5440, 5600] dup_rows_per_dst=[2514, 2468, 2622, 2810, 2466, 2894, 2542, 2718] combine_rows_per_src=[5486, 5410, 5598, 5814, 5484, 5670, 5440, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160169984 dispatch_meta_bytes=229024 combine_payload_bytes=160169984 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160399008 ingress_bytes=210501632 link_bytes=210501632 token_bytes=320568992 weight_movement_bytes=50331648 total_nvlink_bytes=370900640 weight_to_token_ratio=0.157007 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601674198e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44540 payload_rows_remote=38938 dup_rows=20996 dup_rows_remote=18360 dedup_ratio=0.320374 router_tokens_per_expert=[515, 517, 548, 504, 495, 502, 499, 540, 521, 522, 535, 495, 483, 520, 529, 514, 516, 490, 529, 483, 491, 502, 489, 505, 497, 519, 522, 528, 491, 518, 488, 504, 499, 517, 526, 512, 495, 531, 534, 534, 539, 479, 510, 498, 500, 522, 456, 499, 524, 512, 524, 498, 503, 507, 513, 513, 529, 496, 492, 540, 540, 561, 497, 557] payload_rows_per_dst=[5538, 5602, 5538, 5534, 5588, 5518, 5544, 5678] dup_rows_per_dst=[2702, 2636, 2472, 2600, 2708, 2488, 2644, 2746] combine_rows_per_src=[5538, 5602, 5538, 5534, 5588, 5518, 5544, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38892 dup_rows=21108 dup_rows_remote=18516 dedup_ratio=0.322083 router_tokens_per_expert=[521, 526, 500, 494, 485, 493, 536, 509, 476, 533, 542, 490, 506, 552, 505, 550, 485, 508, 491, 514, 556, 505, 493, 524, 487, 520, 504, 510, 496, 532, 457, 496, 475, 483, 515, 496, 555, 525, 511, 532, 494, 510, 554, 495, 537, 470, 463, 501, 537, 527, 542, 506, 513, 555, 504, 525, 525, 525, 487, 534, 514, 527, 510, 525] payload_rows_per_dst=[5536, 5654, 5536, 5494, 5560, 5444, 5632, 5572] dup_rows_per_dst=[2592, 2712, 2616, 2510, 2644, 2604, 2786, 2644] combine_rows_per_src=[5536, 5654, 5536, 5494, 5560, 5444, 5632, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158875648 dispatch_meta_bytes=229128 combine_payload_bytes=158875648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159104776 ingress_bytes=184041472 link_bytes=184041472 token_bytes=317980424 weight_movement_bytes=25165824 total_nvlink_bytes=343146248 weight_to_token_ratio=0.079143 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400342954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44338 payload_rows_remote=38922 dup_rows=21198 dup_rows_remote=18656 dedup_ratio=0.323456 router_tokens_per_expert=[511, 479, 518, 506, 538, 480, 506, 513, 536, 514, 509, 546, 537, 516, 515, 506, 502, 550, 501, 567, 511, 511, 517, 476, 536, 491, 496, 545, 509, 539, 472, 541, 516, 542, 498, 509, 541, 464, 513, 553, 509, 490, 495, 528, 500, 496, 514, 511, 507, 497, 496, 502, 467, 495, 487, 528, 531, 537, 542, 511, 472, 480, 508, 535] payload_rows_per_dst=[5506, 5630, 5590, 5540, 5530, 5530, 5416, 5596] dup_rows_per_dst=[2596, 2728, 2680, 2718, 2742, 2556, 2542, 2636] combine_rows_per_src=[5506, 5630, 5590, 5540, 5530, 5530, 5416, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44532 payload_rows_remote=39038 dup_rows=21004 dup_rows_remote=18380 dedup_ratio=0.320496 router_tokens_per_expert=[504, 511, 513, 532, 499, 529, 496, 495, 510, 492, 501, 483, 535, 514, 511, 481, 520, 554, 496, 488, 541, 465, 477, 518, 535, 494, 474, 525, 566, 520, 514, 536, 506, 504, 513, 479, 489, 537, 543, 515, 592, 503, 481, 532, 507, 533, 513, 525, 487, 492, 534, 528, 517, 535, 503, 486, 522, 485, 521, 512, 498, 499, 522, 526] payload_rows_per_dst=[5558, 5462, 5494, 5622, 5596, 5718, 5550, 5532] dup_rows_per_dst=[2600, 2592, 2624, 2706, 2576, 2654, 2614, 2638] combine_rows_per_src=[5558, 5462, 5494, 5622, 5596, 5718, 5550, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=228872 combine_payload_bytes=159342592 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159571464 ingress_bytes=209674240 link_bytes=209674240 token_bytes=318914056 weight_movement_bytes=50331648 total_nvlink_bytes=369245704 weight_to_token_ratio=0.157822 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595378701e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159490048 dispatch_meta_bytes=229192 combine_payload_bytes=159490048 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159719240 ingress_bytes=209821696 link_bytes=209821696 token_bytes=319209288 weight_movement_bytes=50331648 total_nvlink_bytes=369540936 weight_to_token_ratio=0.157676 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596500671e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159301632 dispatch_meta_bytes=229632 combine_payload_bytes=159301632 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159531264 ingress_bytes=184467456 link_bytes=184467456 token_bytes=318832896 weight_movement_bytes=25165824 total_nvlink_bytes=343998720 weight_to_token_ratio=0.078931 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403584200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=38940 dup_rows=20976 dup_rows_remote=18218 dedup_ratio=0.320068 router_tokens_per_expert=[512, 506, 509, 495, 483, 499, 498, 508, 500, 476, 507, 542, 526, 492, 510, 534, 508, 524, 497, 512, 500, 499, 505, 523, 503, 493, 564, 535, 529, 537, 501, 527, 530, 486, 528, 514, 522, 509, 510, 539, 521, 490, 511, 473, 539, 523, 525, 502, 486, 562, 487, 541, 514, 505, 523, 512, 509, 499, 497, 515, 511, 557, 455, 519] payload_rows_per_dst=[5538, 5658, 5498, 5620, 5606, 5542, 5558, 5540] dup_rows_per_dst=[2482, 2516, 2638, 2758, 2670, 2626, 2702, 2584] combine_rows_per_src=[5538, 5658, 5498, 5620, 5606, 5542, 5558, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159899648 dispatch_meta_bytes=229672 combine_payload_bytes=159899648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160129320 ingress_bytes=185065472 link_bytes=185065472 token_bytes=320028968 weight_movement_bytes=25165824 total_nvlink_bytes=345194792 weight_to_token_ratio=0.078636 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408134411e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=230312 combine_payload_bytes=159424512 prefetch_bytes=0 reduce_bytes=0 egress_bytes=159654824 ingress_bytes=159424512 link_bytes=159654824 token_bytes=319079336 weight_movement_bytes=0 total_nvlink_bytes=319079336 weight_to_token_ratio=0.000000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.214788740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44630 payload_rows_remote=39050 dup_rows=20906 dup_rows_remote=18234 dedup_ratio=0.319000 router_tokens_per_expert=[487, 522, 545, 559, 539, 474, 517, 510, 495, 540, 548, 492, 508, 531, 511, 496, 511, 476, 527, 495, 526, 506, 477, 461, 491, 483, 525, 474, 531, 497, 506, 520, 516, 517, 515, 536, 532, 482, 509, 519, 529, 543, 492, 536, 537, 518, 494, 515, 481, 505, 540, 525, 535, 527, 535, 527, 492, 519, 479, 525, 502, 496, 502, 508] payload_rows_per_dst=[5596, 5584, 5484, 5546, 5580, 5654, 5632, 5554] dup_rows_per_dst=[2710, 2658, 2474, 2508, 2672, 2674, 2718, 2492] combine_rows_per_src=[5596, 5584, 5484, 5546, 5580, 5654, 5632, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=228632 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159726872 ingress_bytes=209829888 link_bytes=209829888 token_bytes=319225112 weight_movement_bytes=50331648 total_nvlink_bytes=369556760 weight_to_token_ratio=0.157668 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596563003e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44306 payload_rows_remote=38684 dup_rows=21230 dup_rows_remote=18404 dedup_ratio=0.323944 router_tokens_per_expert=[479, 533, 500, 536, 483, 522, 518, 477, 465, 494, 508, 515, 530, 528, 532, 494, 520, 507, 485, 492, 508, 528, 464, 488, 461, 494, 508, 537, 495, 533, 516, 507, 550, 530, 501, 493, 486, 547, 544, 529, 525, 545, 515, 580, 490, 530, 526, 513, 498, 497, 511, 513, 531, 497, 573, 511, 488, 486, 531, 493, 528, 484, 526, 540] payload_rows_per_dst=[5546, 5560, 5460, 5458, 5554, 5622, 5580, 5526] dup_rows_per_dst=[2550, 2572, 2524, 2644, 2806, 2826, 2682, 2626] combine_rows_per_src=[5546, 5560, 5460, 5458, 5554, 5622, 5580, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44338 payload_rows_remote=38754 dup_rows=21198 dup_rows_remote=18466 dedup_ratio=0.323456 router_tokens_per_expert=[507, 469, 494, 538, 490, 509, 509, 505, 500, 559, 501, 488, 491, 481, 501, 502, 531, 485, 487, 485, 521, 512, 509, 503, 524, 506, 512, 536, 534, 494, 525, 505, 519, 542, 526, 530, 525, 539, 507, 518, 518, 530, 525, 489, 503, 503, 481, 540, 527, 529, 519, 521, 505, 517, 480, 505, 553, 547, 495, 543, 525, 504, 497, 493] payload_rows_per_dst=[5480, 5514, 5512, 5600, 5532, 5526, 5590, 5584] dup_rows_per_dst=[2562, 2532, 2554, 2582, 2880, 2650, 2706, 2732] combine_rows_per_src=[5480, 5514, 5512, 5600, 5532, 5526, 5590, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159948800 dispatch_meta_bytes=229136 combine_payload_bytes=159948800 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160177936 ingress_bytes=235446272 link_bytes=235446272 token_bytes=320126736 weight_movement_bytes=75497472 total_nvlink_bytes=395624208 weight_to_token_ratio=0.235836 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791474087e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44696 payload_rows_remote=39128 dup_rows=20840 dup_rows_remote=18192 dedup_ratio=0.317993 router_tokens_per_expert=[509, 535, 536, 487, 517, 510, 468, 532, 517, 543, 507, 499, 469, 537, 534, 502, 505, 539, 482, 474, 514, 553, 518, 519, 505, 538, 513, 481, 496, 497, 547, 533, 524, 517, 529, 504, 511, 501, 518, 486, 501, 499, 529, 501, 497, 521, 514, 531, 555, 517, 497, 482, 508, 498, 503, 515, 505, 513, 523, 496, 518, 532, 509, 498] payload_rows_per_dst=[5652, 5568, 5622, 5616, 5564, 5552, 5580, 5542] dup_rows_per_dst=[2536, 2648, 2586, 2604, 2616, 2634, 2570, 2646] combine_rows_per_src=[5652, 5568, 5622, 5616, 5564, 5552, 5580, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44314 payload_rows_remote=38696 dup_rows=21222 dup_rows_remote=18442 dedup_ratio=0.323822 router_tokens_per_expert=[511, 530, 515, 512, 496, 516, 519, 497, 513, 472, 499, 526, 501, 521, 518, 498, 572, 495, 472, 474, 498, 544, 531, 516, 518, 488, 547, 490, 503, 514, 501, 515, 522, 531, 497, 499, 480, 512, 500, 523, 529, 543, 521, 494, 518, 480, 523, 479, 537, 516, 537, 501, 473, 518, 502, 520, 501, 531, 531, 541, 525, 516, 517, 529] payload_rows_per_dst=[5618, 5600, 5744, 5556, 5460, 5310, 5416, 5610] dup_rows_per_dst=[2780, 2782, 2994, 2614, 2370, 2330, 2586, 2766] combine_rows_per_src=[5618, 5600, 5744, 5556, 5460, 5310, 5416, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158449664 dispatch_meta_bytes=228352 combine_payload_bytes=158449664 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158678016 ingress_bytes=208781312 link_bytes=208781312 token_bytes=317127680 weight_movement_bytes=50331648 total_nvlink_bytes=367459328 weight_to_token_ratio=0.158711 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588584551e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44386 payload_rows_remote=38848 dup_rows=21150 dup_rows_remote=18436 dedup_ratio=0.322723 router_tokens_per_expert=[524, 544, 525, 514, 509, 523, 500, 523, 517, 507, 509, 518, 513, 527, 490, 517, 523, 490, 564, 498, 523, 544, 478, 506, 479, 492, 534, 541, 525, 543, 538, 494, 530, 455, 508, 501, 474, 549, 531, 536, 524, 527, 471, 527, 510, 505, 476, 497, 507, 526, 520, 536, 492, 488, 492, 505, 493, 510, 508, 481, 551, 478, 490, 538] payload_rows_per_dst=[5624, 5564, 5538, 5658, 5516, 5456, 5532, 5498] dup_rows_per_dst=[2700, 2632, 2714, 2634, 2652, 2618, 2600, 2600] combine_rows_per_src=[5624, 5564, 5538, 5658, 5516, 5456, 5532, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44518 payload_rows_remote=39022 dup_rows=21018 dup_rows_remote=18462 dedup_ratio=0.320709 router_tokens_per_expert=[540, 527, 540, 488, 537, 534, 499, 499, 503, 510, 505, 519, 519, 537, 508, 544, 501, 547, 520, 516, 525, 498, 546, 511, 524, 484, 494, 497, 490, 522, 524, 516, 508, 517, 536, 507, 476, 508, 481, 494, 510, 525, 498, 512, 518, 506, 507, 533, 499, 486, 488, 476, 493, 533, 558, 493, 522, 510, 518, 519, 512, 506, 496, 499] payload_rows_per_dst=[5680, 5600, 5620, 5514, 5492, 5592, 5496, 5524] dup_rows_per_dst=[2648, 2690, 2708, 2588, 2562, 2626, 2556, 2640] combine_rows_per_src=[5680, 5600, 5620, 5514, 5492, 5592, 5496, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158736384 dispatch_meta_bytes=228880 combine_payload_bytes=158736384 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158965264 ingress_bytes=209068032 link_bytes=209068032 token_bytes=317701648 weight_movement_bytes=50331648 total_nvlink_bytes=368033296 weight_to_token_ratio=0.158424 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590766159e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44322 payload_rows_remote=38688 dup_rows=21214 dup_rows_remote=18588 dedup_ratio=0.323700 router_tokens_per_expert=[497, 525, 505, 525, 504, 504, 520, 531, 486, 543, 482, 509, 516, 507, 492, 520, 484, 477, 537, 526, 570, 508, 532, 535, 533, 537, 497, 520, 514, 486, 551, 492, 528, 515, 500, 495, 491, 483, 534, 545, 525, 523, 483, 532, 497, 531, 508, 533, 464, 515, 503, 491, 454, 538, 500, 506, 517, 545, 521, 477, 539, 499, 490, 521] payload_rows_per_dst=[5522, 5472, 5566, 5634, 5500, 5610, 5436, 5582] dup_rows_per_dst=[2700, 2638, 2772, 2626, 2682, 2654, 2506, 2636] combine_rows_per_src=[5522, 5472, 5566, 5634, 5500, 5610, 5436, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159834112 dispatch_meta_bytes=229936 combine_payload_bytes=159834112 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160064048 ingress_bytes=210165760 link_bytes=210165760 token_bytes=319898160 weight_movement_bytes=50331648 total_nvlink_bytes=370229808 weight_to_token_ratio=0.157336 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599118600e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160268288 dispatch_meta_bytes=229280 combine_payload_bytes=160268288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160497568 ingress_bytes=185434112 link_bytes=185434112 token_bytes=320765856 weight_movement_bytes=25165824 total_nvlink_bytes=345931680 weight_to_token_ratio=0.078455 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410939335e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158498816 dispatch_meta_bytes=228552 combine_payload_bytes=158498816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158727368 ingress_bytes=183664640 link_bytes=183664640 token_bytes=317226184 weight_movement_bytes=25165824 total_nvlink_bytes=342392008 weight_to_token_ratio=0.079331 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397475698e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159121408 dispatch_meta_bytes=229136 combine_payload_bytes=159121408 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159350544 ingress_bytes=209453056 link_bytes=209453056 token_bytes=318471952 weight_movement_bytes=50331648 total_nvlink_bytes=368803600 weight_to_token_ratio=0.158041 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593695747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44398 payload_rows_remote=38898 dup_rows=21138 dup_rows_remote=18606 dedup_ratio=0.322540 router_tokens_per_expert=[493, 532, 482, 509, 497, 496, 520, 544, 511, 541, 490, 491, 500, 522, 526, 514, 505, 514, 494, 528, 523, 509, 525, 520, 523, 503, 514, 506, 550, 498, 535, 492, 485, 514, 480, 520, 492, 518, 505, 502, 510, 520, 509, 539, 492, 533, 507, 515, 502, 532, 526, 497, 523, 513, 504, 525, 514, 497, 540, 500, 527, 500, 514, 506] payload_rows_per_dst=[5518, 5630, 5510, 5590, 5500, 5560, 5538, 5552] dup_rows_per_dst=[2628, 2560, 2726, 2652, 2532, 2690, 2706, 2644] combine_rows_per_src=[5518, 5630, 5510, 5590, 5500, 5560, 5538, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=39030 dup_rows=20988 dup_rows_remote=18416 dedup_ratio=0.320251 router_tokens_per_expert=[538, 510, 509, 529, 522, 529, 499, 501, 502, 515, 501, 522, 533, 516, 512, 538, 512, 527, 521, 524, 490, 470, 534, 531, 522, 525, 500, 538, 510, 517, 538, 497, 502, 556, 497, 528, 492, 544, 509, 556, 548, 479, 476, 481, 537, 494, 499, 531, 531, 466, 483, 527, 513, 520, 482, 513, 530, 482, 485, 513, 481, 502, 474, 505] payload_rows_per_dst=[5586, 5634, 5588, 5558, 5670, 5518, 5514, 5480] dup_rows_per_dst=[2688, 2644, 2630, 2736, 2698, 2572, 2556, 2464] combine_rows_per_src=[5586, 5634, 5588, 5558, 5670, 5518, 5514, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158466048 dispatch_meta_bytes=229104 combine_payload_bytes=158466048 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158695152 ingress_bytes=208797696 link_bytes=208797696 token_bytes=317161200 weight_movement_bytes=50331648 total_nvlink_bytes=367492848 weight_to_token_ratio=0.158694 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588709214e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44406 payload_rows_remote=38858 dup_rows=21130 dup_rows_remote=18434 dedup_ratio=0.322418 router_tokens_per_expert=[504, 532, 507, 478, 505, 485, 547, 534, 530, 482, 522, 485, 484, 525, 490, 514, 498, 536, 504, 514, 489, 515, 469, 549, 542, 517, 532, 541, 484, 527, 502, 522, 500, 517, 455, 500, 522, 538, 508, 486, 528, 535, 479, 530, 536, 560, 535, 533, 510, 537, 509, 487, 527, 500, 517, 479, 493, 497, 488, 504, 552, 486, 538, 517] payload_rows_per_dst=[5474, 5474, 5572, 5358, 5606, 5660, 5714, 5548] dup_rows_per_dst=[2406, 2590, 2688, 2428, 2614, 2718, 2990, 2696] combine_rows_per_src=[5474, 5474, 5572, 5358, 5606, 5660, 5714, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159326208 dispatch_meta_bytes=230016 combine_payload_bytes=159326208 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159556224 ingress_bytes=184492032 link_bytes=184492032 token_bytes=318882432 weight_movement_bytes=25165824 total_nvlink_bytes=344048256 weight_to_token_ratio=0.078919 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403771195e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=229784 combine_payload_bytes=159866880 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160096664 ingress_bytes=185032704 link_bytes=185032704 token_bytes=319963544 weight_movement_bytes=25165824 total_nvlink_bytes=345129368 weight_to_token_ratio=0.078652 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407885084e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159162368 dispatch_meta_bytes=229168 combine_payload_bytes=159162368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159391536 ingress_bytes=184328192 link_bytes=184328192 token_bytes=318553904 weight_movement_bytes=25165824 total_nvlink_bytes=343719728 weight_to_token_ratio=0.079000 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402524562e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44644 payload_rows_remote=38972 dup_rows=20892 dup_rows_remote=18290 dedup_ratio=0.318787 router_tokens_per_expert=[508, 499, 499, 519, 496, 469, 497, 525, 471, 485, 482, 494, 488, 510, 504, 501, 538, 524, 501, 528, 503, 562, 527, 483, 497, 512, 506, 517, 558, 530, 533, 539, 536, 518, 503, 529, 549, 523, 512, 525, 487, 525, 512, 489, 467, 508, 517, 521, 527, 535, 516, 490, 507, 492, 554, 516, 499, 505, 537, 502, 538, 511, 476, 537] payload_rows_per_dst=[5508, 5498, 5668, 5610, 5582, 5490, 5672, 5616] dup_rows_per_dst=[2516, 2372, 2664, 2774, 2808, 2562, 2602, 2594] combine_rows_per_src=[5508, 5498, 5668, 5610, 5582, 5490, 5672, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=38888 dup_rows=21086 dup_rows_remote=18594 dedup_ratio=0.321747 router_tokens_per_expert=[466, 508, 521, 520, 535, 529, 543, 534, 471, 524, 514, 524, 503, 526, 492, 473, 513, 499, 490, 535, 521, 515, 491, 555, 511, 509, 532, 511, 504, 471, 532, 486, 529, 518, 494, 473, 504, 507, 513, 534, 511, 504, 536, 549, 539, 514, 531, 514, 490, 460, 475, 515, 501, 525, 541, 505, 511, 513, 512, 527, 510, 559, 482, 514] payload_rows_per_dst=[5616, 5562, 5578, 5530, 5478, 5670, 5468, 5548] dup_rows_per_dst=[2696, 2492, 2660, 2582, 2666, 2726, 2556, 2708] combine_rows_per_src=[5616, 5562, 5578, 5530, 5478, 5670, 5468, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38946 dup_rows=21068 dup_rows_remote=18508 dedup_ratio=0.321472 router_tokens_per_expert=[538, 510, 539, 503, 500, 480, 497, 513, 494, 487, 504, 545, 493, 555, 531, 533, 469, 510, 512, 533, 504, 500, 500, 513, 485, 483, 492, 521, 553, 509, 501, 497, 483, 507, 465, 487, 550, 538, 537, 526, 529, 547, 517, 503, 524, 541, 483, 537, 470, 556, 525, 509, 523, 470, 536, 507, 497, 472, 537, 547, 503, 513, 511, 514] payload_rows_per_dst=[5506, 5590, 5522, 5506, 5554, 5624, 5564, 5602] dup_rows_per_dst=[2654, 2694, 2560, 2576, 2632, 2738, 2628, 2586] combine_rows_per_src=[5506, 5590, 5522, 5506, 5554, 5624, 5564, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38784 dup_rows=21108 dup_rows_remote=18398 dedup_ratio=0.322083 router_tokens_per_expert=[549, 537, 511, 499, 518, 534, 524, 505, 503, 543, 529, 496, 462, 519, 535, 533, 509, 518, 513, 522, 524, 499, 486, 527, 530, 507, 513, 502, 543, 515, 484, 489, 477, 492, 523, 529, 492, 515, 495, 493, 506, 492, 480, 476, 516, 508, 534, 504, 522, 499, 513, 561, 494, 511, 489, 484, 523, 514, 532, 536, 502, 536, 494, 548] payload_rows_per_dst=[5644, 5788, 5268, 5570, 5432, 5488, 5516, 5722] dup_rows_per_dst=[2710, 2986, 2394, 2596, 2600, 2544, 2630, 2648] combine_rows_per_src=[5644, 5788, 5268, 5570, 5432, 5488, 5516, 5722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44566 payload_rows_remote=39006 dup_rows=20970 dup_rows_remote=18412 dedup_ratio=0.319977 router_tokens_per_expert=[505, 483, 480, 507, 502, 509, 533, 519, 459, 517, 525, 520, 481, 522, 514, 497, 523, 515, 510, 509, 512, 517, 516, 521, 555, 493, 520, 501, 490, 497, 478, 525, 533, 480, 511, 513, 546, 504, 572, 511, 525, 487, 544, 531, 526, 482, 492, 543, 486, 515, 523, 525, 513, 506, 544, 532, 544, 490, 501, 542, 483, 514, 501, 494] payload_rows_per_dst=[5592, 5514, 5568, 5560, 5608, 5586, 5680, 5458] dup_rows_per_dst=[2484, 2556, 2678, 2558, 2732, 2674, 2608, 2680] combine_rows_per_src=[5592, 5514, 5568, 5560, 5608, 5586, 5680, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=229048 combine_payload_bytes=159629312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159858360 ingress_bytes=184795136 link_bytes=184795136 token_bytes=319487672 weight_movement_bytes=25165824 total_nvlink_bytes=344653496 weight_to_token_ratio=0.078769 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406077466e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38938 dup_rows=21092 dup_rows_remote=18560 dedup_ratio=0.321838 router_tokens_per_expert=[511, 507, 522, 522, 505, 530, 518, 492, 496, 489, 505, 518, 543, 504, 493, 537, 528, 519, 514, 545, 505, 489, 499, 483, 499, 537, 499, 496, 497, 510, 511, 491, 489, 506, 524, 510, 494, 512, 512, 472, 544, 546, 547, 483, 515, 560, 492, 489, 535, 531, 512, 517, 529, 477, 542, 539, 493, 491, 493, 507, 517, 527, 507, 542] payload_rows_per_dst=[5548, 5528, 5598, 5560, 5506, 5550, 5630, 5524] dup_rows_per_dst=[2666, 2642, 2566, 2520, 2532, 2802, 2734, 2630] combine_rows_per_src=[5548, 5528, 5598, 5560, 5506, 5550, 5630, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=229928 combine_payload_bytes=159285248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159515176 ingress_bytes=184451072 link_bytes=184451072 token_bytes=318800424 weight_movement_bytes=25165824 total_nvlink_bytes=343966248 weight_to_token_ratio=0.078939 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403459536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158859264 dispatch_meta_bytes=228728 combine_payload_bytes=158859264 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159087992 ingress_bytes=234356736 link_bytes=234356736 token_bytes=317947256 weight_movement_bytes=75497472 total_nvlink_bytes=393444728 weight_to_token_ratio=0.237453 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783183977e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229816 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159752632 ingress_bytes=184688640 link_bytes=184688640 token_bytes=319275448 weight_movement_bytes=25165824 total_nvlink_bytes=344441272 weight_to_token_ratio=0.078822 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405267154e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44402 payload_rows_remote=38852 dup_rows=21134 dup_rows_remote=18396 dedup_ratio=0.322479 router_tokens_per_expert=[528, 544, 512, 508, 466, 505, 542, 518, 480, 490, 492, 489, 535, 529, 503, 529, 506, 499, 498, 494, 572, 516, 511, 520, 503, 532, 486, 516, 532, 520, 506, 480, 543, 494, 511, 508, 520, 495, 501, 502, 535, 519, 522, 493, 528, 524, 522, 501, 536, 548, 556, 495, 483, 504, 477, 529, 533, 501, 501, 484, 503, 539, 477, 523] payload_rows_per_dst=[5526, 5494, 5612, 5504, 5586, 5550, 5582, 5548] dup_rows_per_dst=[2720, 2600, 2620, 2646, 2562, 2738, 2674, 2574] combine_rows_per_src=[5526, 5494, 5612, 5504, 5586, 5550, 5582, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38814 dup_rows=21034 dup_rows_remote=18194 dedup_ratio=0.320953 router_tokens_per_expert=[517, 512, 488, 531, 533, 476, 519, 536, 529, 499, 531, 522, 485, 520, 465, 510, 530, 539, 495, 516, 491, 490, 488, 480, 524, 484, 517, 555, 517, 516, 533, 532, 504, 498, 524, 501, 553, 519, 499, 492, 527, 523, 526, 528, 496, 500, 512, 517, 534, 543, 485, 508, 512, 506, 522, 534, 480, 482, 490, 484, 512, 521, 530, 526] payload_rows_per_dst=[5560, 5516, 5488, 5566, 5660, 5418, 5606, 5688] dup_rows_per_dst=[2628, 2606, 2570, 2736, 2610, 2318, 2726, 2840] combine_rows_per_src=[5560, 5516, 5488, 5566, 5660, 5418, 5606, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159768576 dispatch_meta_bytes=229672 combine_payload_bytes=159768576 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159998248 ingress_bytes=210100224 link_bytes=210100224 token_bytes=319766824 weight_movement_bytes=50331648 total_nvlink_bytes=370098472 weight_to_token_ratio=0.157401 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598619947e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159490048 dispatch_meta_bytes=229992 combine_payload_bytes=159490048 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159720040 ingress_bytes=209821696 link_bytes=209821696 token_bytes=319210088 weight_movement_bytes=50331648 total_nvlink_bytes=369541736 weight_to_token_ratio=0.157676 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596500671e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159137792 dispatch_meta_bytes=228992 combine_payload_bytes=159137792 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159366784 ingress_bytes=234635264 link_bytes=234635264 token_bytes=318504576 weight_movement_bytes=75497472 total_nvlink_bytes=394002048 weight_to_token_ratio=0.237037 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785303253e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158982144 dispatch_meta_bytes=228032 combine_payload_bytes=158982144 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159210176 ingress_bytes=184147968 link_bytes=184147968 token_bytes=318192320 weight_movement_bytes=25165824 total_nvlink_bytes=343358144 weight_to_token_ratio=0.079090 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401153265e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38908 dup_rows=21064 dup_rows_remote=18480 dedup_ratio=0.321411 router_tokens_per_expert=[520, 508, 540, 496, 486, 516, 498, 486, 498, 521, 527, 503, 497, 462, 518, 507, 556, 494, 468, 487, 539, 504, 485, 502, 539, 521, 495, 514, 508, 519, 542, 483, 503, 505, 540, 529, 533, 554, 497, 536, 524, 488, 498, 513, 518, 509, 496, 490, 519, 501, 515, 511, 513, 511, 479, 525, 519, 525, 471, 589, 551, 485, 553, 529] payload_rows_per_dst=[5476, 5494, 5502, 5630, 5700, 5478, 5564, 5628] dup_rows_per_dst=[2624, 2572, 2568, 2612, 2694, 2594, 2584, 2816] combine_rows_per_src=[5476, 5494, 5502, 5630, 5700, 5478, 5564, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44306 payload_rows_remote=38768 dup_rows=21230 dup_rows_remote=18700 dedup_ratio=0.323944 router_tokens_per_expert=[484, 512, 550, 544, 497, 515, 532, 499, 534, 481, 484, 481, 526, 490, 521, 517, 535, 507, 513, 523, 477, 540, 493, 474, 541, 521, 538, 533, 518, 503, 534, 523, 500, 553, 527, 483, 530, 524, 528, 529, 491, 523, 519, 501, 522, 512, 476, 498, 536, 468, 529, 516, 502, 542, 496, 498, 508, 510, 519, 519, 468, 526, 473, 502] payload_rows_per_dst=[5536, 5538, 5536, 5622, 5550, 5494, 5546, 5484] dup_rows_per_dst=[2730, 2530, 2588, 2800, 2798, 2590, 2628, 2566] combine_rows_per_src=[5536, 5538, 5536, 5622, 5550, 5494, 5546, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=38968 dup_rows=20978 dup_rows_remote=18282 dedup_ratio=0.320099 router_tokens_per_expert=[496, 485, 528, 531, 521, 509, 538, 534, 531, 510, 516, 541, 497, 515, 474, 509, 514, 537, 502, 514, 496, 474, 536, 535, 495, 572, 491, 524, 524, 482, 478, 504, 547, 484, 501, 524, 501, 451, 539, 531, 522, 554, 519, 501, 498, 538, 528, 532, 537, 514, 501, 513, 512, 467, 541, 544, 462, 487, 477, 471, 498, 518, 536, 507] payload_rows_per_dst=[5590, 5826, 5710, 5542, 5428, 5664, 5350, 5448] dup_rows_per_dst=[2696, 2836, 2850, 2598, 2384, 2718, 2432, 2464] combine_rows_per_src=[5590, 5826, 5710, 5542, 5428, 5664, 5350, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38868 dup_rows=21064 dup_rows_remote=18492 dedup_ratio=0.321411 router_tokens_per_expert=[507, 514, 509, 476, 555, 532, 530, 489, 528, 550, 517, 522, 553, 513, 515, 502, 493, 509, 548, 491, 501, 495, 525, 526, 512, 486, 509, 524, 514, 538, 534, 534, 498, 486, 520, 519, 510, 522, 513, 492, 496, 497, 529, 483, 507, 489, 483, 516, 501, 469, 530, 501, 535, 546, 536, 505, 515, 510, 509, 527, 483, 508, 482, 500] payload_rows_per_dst=[5528, 5626, 5604, 5606, 5478, 5490, 5598, 5542] dup_rows_per_dst=[2696, 2774, 2572, 2696, 2642, 2510, 2648, 2526] combine_rows_per_src=[5528, 5626, 5604, 5606, 5478, 5490, 5598, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44702 payload_rows_remote=39174 dup_rows=20834 dup_rows_remote=18316 dedup_ratio=0.317902 router_tokens_per_expert=[510, 523, 532, 513, 528, 502, 523, 497, 515, 543, 537, 510, 534, 497, 502, 507, 513, 468, 554, 515, 507, 550, 512, 512, 529, 496, 532, 502, 483, 511, 495, 475, 547, 492, 484, 502, 488, 521, 485, 512, 537, 515, 512, 498, 533, 507, 523, 485, 520, 506, 508, 527, 534, 529, 486, 541, 485, 473, 513, 522, 522, 520, 509, 505] payload_rows_per_dst=[5610, 5582, 5532, 5528, 5504, 5676, 5674, 5596] dup_rows_per_dst=[2646, 2708, 2730, 2518, 2558, 2544, 2628, 2502] combine_rows_per_src=[5610, 5582, 5532, 5528, 5504, 5676, 5674, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=38956 dup_rows=21086 dup_rows_remote=18476 dedup_ratio=0.321747 router_tokens_per_expert=[482, 511, 497, 485, 508, 540, 508, 493, 519, 521, 507, 537, 518, 504, 500, 543, 517, 510, 510, 531, 509, 505, 514, 558, 522, 480, 497, 526, 481, 477, 494, 545, 473, 475, 525, 508, 504, 508, 523, 536, 516, 512, 489, 522, 531, 540, 547, 518, 478, 532, 468, 512, 535, 499, 544, 516, 526, 503, 495, 499, 505, 533, 529, 518] payload_rows_per_dst=[5512, 5596, 5586, 5528, 5494, 5630, 5622, 5482] dup_rows_per_dst=[2536, 2702, 2722, 2516, 2610, 2720, 2546, 2734] combine_rows_per_src=[5512, 5596, 5586, 5528, 5494, 5630, 5622, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159367168 dispatch_meta_bytes=229552 combine_payload_bytes=159367168 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159596720 ingress_bytes=209698816 link_bytes=209698816 token_bytes=318963888 weight_movement_bytes=50331648 total_nvlink_bytes=369295536 weight_to_token_ratio=0.157797 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595565696e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158793728 dispatch_meta_bytes=229872 combine_payload_bytes=158793728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159023600 ingress_bytes=183959552 link_bytes=183959552 token_bytes=317817328 weight_movement_bytes=25165824 total_nvlink_bytes=342983152 weight_to_token_ratio=0.079183 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399719637e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159612928 dispatch_meta_bytes=229000 combine_payload_bytes=159612928 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159841928 ingress_bytes=209944576 link_bytes=209944576 token_bytes=319454856 weight_movement_bytes=50331648 total_nvlink_bytes=369786504 weight_to_token_ratio=0.157555 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597435646e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159203328 dispatch_meta_bytes=229440 combine_payload_bytes=159203328 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159432768 ingress_bytes=234700800 link_bytes=234700800 token_bytes=318636096 weight_movement_bytes=75497472 total_nvlink_bytes=394133568 weight_to_token_ratio=0.236939 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785801906e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44574 payload_rows_remote=38982 dup_rows=20962 dup_rows_remote=18424 dedup_ratio=0.319855 router_tokens_per_expert=[522, 502, 519, 482, 480, 484, 528, 515, 505, 480, 491, 497, 548, 515, 525, 511, 496, 536, 545, 509, 508, 517, 531, 507, 497, 491, 495, 509, 497, 490, 485, 538, 535, 464, 499, 502, 481, 485, 500, 549, 543, 510, 504, 551, 469, 519, 510, 459, 543, 541, 516, 488, 525, 498, 559, 517, 510, 550, 542, 509, 551, 532, 522, 530] payload_rows_per_dst=[5474, 5540, 5642, 5504, 5506, 5592, 5612, 5704] dup_rows_per_dst=[2590, 2604, 2656, 2500, 2524, 2538, 2762, 2788] combine_rows_per_src=[5474, 5540, 5642, 5504, 5506, 5592, 5612, 5704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44240 payload_rows_remote=38628 dup_rows=21296 dup_rows_remote=18516 dedup_ratio=0.324951 router_tokens_per_expert=[509, 477, 487, 526, 482, 484, 528, 507, 496, 514, 545, 479, 492, 528, 509, 540, 489, 524, 481, 556, 523, 552, 507, 483, 479, 494, 506, 473, 524, 529, 537, 523, 471, 521, 520, 486, 519, 535, 558, 564, 517, 505, 501, 503, 519, 507, 515, 550, 512, 501, 537, 492, 514, 522, 485, 505, 511, 546, 532, 535, 496, 523, 470, 513] payload_rows_per_dst=[5488, 5512, 5372, 5524, 5720, 5574, 5438, 5612] dup_rows_per_dst=[2512, 2554, 2450, 2606, 3020, 2660, 2714, 2780] combine_rows_per_src=[5488, 5512, 5372, 5524, 5720, 5574, 5438, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160456704 dispatch_meta_bytes=229960 combine_payload_bytes=160456704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160686664 ingress_bytes=185622528 link_bytes=185622528 token_bytes=321143368 weight_movement_bytes=25165824 total_nvlink_bytes=346309192 weight_to_token_ratio=0.078363 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412372963e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=229728 combine_payload_bytes=159563776 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159793504 ingress_bytes=184729600 link_bytes=184729600 token_bytes=319357280 weight_movement_bytes=25165824 total_nvlink_bytes=344523104 weight_to_token_ratio=0.078801 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405578813e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5103503784 prefetch_bytes=327155712 reduce_bytes=478150656 weight_movement_bytes=805306368 egress_bytes=2553588136 ingress_bytes=3355222016 link_bytes=3355222016 total_nvlink_bytes=5908810152 weight_to_token_ratio=0.157795 prefetch_share_of_ingress=0.097506 dedup_ratio=0.321545 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=19 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.595585175e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229624 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159899896 ingress_bytes=184836096 link_bytes=184836096 token_bytes=319570168 weight_movement_bytes=25165824 total_nvlink_bytes=344735992 weight_to_token_ratio=0.078749 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406389124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158220288 dispatch_meta_bytes=228576 combine_payload_bytes=158220288 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158448864 ingress_bytes=208551936 link_bytes=208551936 token_bytes=316669152 weight_movement_bytes=50331648 total_nvlink_bytes=367000800 weight_to_token_ratio=0.158941 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.586839265e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5106581096 prefetch_bytes=402653184 reduce_bytes=226492416 weight_movement_bytes=629145600 egress_bytes=2555125352 ingress_bytes=3180601344 link_bytes=3180601344 total_nvlink_bytes=5735726696 weight_to_token_ratio=0.123203 prefetch_share_of_ingress=0.126597 dedup_ratio=0.321005 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=9 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=20320 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.512543828e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5108287680 prefetch_bytes=352321536 reduce_bytes=327155712 weight_movement_bytes=679477248 egress_bytes=2555979968 ingress_bytes=3231784960 link_bytes=3231784960 total_nvlink_bytes=5787764928 weight_to_token_ratio=0.133015 prefetch_share_of_ingress=0.109018 dedup_ratio=0.321121 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=13 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.536884339e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44634 payload_rows_remote=39092 dup_rows=20902 dup_rows_remote=18294 dedup_ratio=0.318939 router_tokens_per_expert=[511, 492, 514, 551, 502, 507, 471, 515, 505, 538, 499, 513, 481, 516, 508, 526, 519, 497, 503, 564, 531, 493, 522, 561, 513, 505, 511, 535, 494, 517, 504, 529, 495, 526, 565, 509, 500, 523, 498, 498, 528, 485, 494, 478, 491, 529, 531, 518, 519, 515, 512, 483, 488, 523, 515, 520, 505, 532, 509, 475, 498, 500, 547, 512] payload_rows_per_dst=[5544, 5562, 5722, 5604, 5562, 5482, 5542, 5616] dup_rows_per_dst=[2582, 2610, 2658, 2612, 2666, 2626, 2608, 2540] combine_rows_per_src=[5544, 5562, 5722, 5604, 5562, 5482, 5542, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38860 dup_rows=21102 dup_rows_remote=18440 dedup_ratio=0.321991 router_tokens_per_expert=[471, 487, 517, 552, 519, 534, 536, 547, 536, 487, 542, 486, 508, 528, 529, 502, 499, 494, 494, 568, 477, 509, 515, 496, 521, 494, 525, 530, 498, 527, 535, 481, 517, 487, 501, 518, 462, 503, 497, 519, 511, 482, 500, 488, 517, 517, 530, 529, 579, 532, 529, 528, 518, 476, 484, 536, 515, 519, 496, 507, 543, 467, 503, 514] payload_rows_per_dst=[5584, 5574, 5514, 5578, 5486, 5606, 5586, 5506] dup_rows_per_dst=[2742, 2662, 2590, 2644, 2522, 2542, 2778, 2622] combine_rows_per_src=[5584, 5574, 5514, 5578, 5486, 5606, 5586, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=38776 dup_rows=21012 dup_rows_remote=18268 dedup_ratio=0.320618 router_tokens_per_expert=[471, 518, 493, 518, 530, 524, 527, 520, 505, 524, 523, 539, 490, 517, 477, 517, 521, 529, 542, 504, 524, 491, 516, 484, 480, 486, 525, 572, 534, 487, 506, 556, 531, 543, 525, 483, 523, 540, 489, 488, 528, 533, 476, 505, 531, 472, 491, 511, 521, 504, 488, 514, 514, 525, 428, 536, 533, 529, 493, 512, 493, 488, 539, 532] payload_rows_per_dst=[5748, 5474, 5580, 5660, 5632, 5558, 5408, 5464] dup_rows_per_dst=[2744, 2456, 2606, 2912, 2612, 2788, 2372, 2522] combine_rows_per_src=[5748, 5474, 5580, 5660, 5632, 5558, 5408, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5108614344 prefetch_bytes=251658240 reduce_bytes=427819008 weight_movement_bytes=679477248 egress_bytes=2556142792 ingress_bytes=3231948800 link_bytes=3231948800 total_nvlink_bytes=5788091592 weight_to_token_ratio=0.133006 prefetch_share_of_ingress=0.077866 dedup_ratio=0.321037 prefetch_slots=20 unused_prefetch_slots=236 slot_utilization=0.078125 reduce_slots=17 remote_demand=10 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=19558 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.536962254e-04 prefetch_bytes_per_gemm_flop=1.196767770e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5096840968 prefetch_bytes=352321536 reduce_bytes=327155712 weight_movement_bytes=679477248 egress_bytes=2550251272 ingress_bytes=3226066944 link_bytes=3226066944 total_nvlink_bytes=5776318216 weight_to_token_ratio=0.133313 prefetch_share_of_ingress=0.109211 dedup_ratio=0.321449 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=13 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.534165121e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160120832 dispatch_meta_bytes=229544 combine_payload_bytes=160120832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160350376 ingress_bytes=210452480 link_bytes=210452480 token_bytes=320471208 weight_movement_bytes=50331648 total_nvlink_bytes=370802856 weight_to_token_ratio=0.157055 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601300208e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159170560 dispatch_meta_bytes=229200 combine_payload_bytes=159170560 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159399760 ingress_bytes=234668032 link_bytes=234668032 token_bytes=318570320 weight_movement_bytes=75497472 total_nvlink_bytes=394067792 weight_to_token_ratio=0.236988 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785552580e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=228176 combine_payload_bytes=158826496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159054672 ingress_bytes=183992320 link_bytes=183992320 token_bytes=317881168 weight_movement_bytes=25165824 total_nvlink_bytes=343046992 weight_to_token_ratio=0.079167 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399968964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5112578288 prefetch_bytes=327155712 reduce_bytes=478150656 weight_movement_bytes=805306368 egress_bytes=2558124272 ingress_bytes=3359760384 link_bytes=3359990696 total_nvlink_bytes=5917884656 weight_to_token_ratio=0.157515 prefetch_share_of_ingress=0.097375 dedup_ratio=0.320318 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=19 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.597852934e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5093086712 prefetch_bytes=352321536 reduce_bytes=251658240 weight_movement_bytes=603979776 egress_bytes=2548372984 ingress_bytes=3148693504 link_bytes=3148693504 total_nvlink_bytes=5697066488 weight_to_token_ratio=0.118588 prefetch_share_of_ingress=0.111895 dedup_ratio=0.321421 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=10 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.497369966e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5109795752 prefetch_bytes=377487360 reduce_bytes=226492416 weight_movement_bytes=603979776 egress_bytes=2556734376 ingress_bytes=3157041152 link_bytes=3157041152 total_nvlink_bytes=5713775528 weight_to_token_ratio=0.118200 prefetch_share_of_ingress=0.119570 dedup_ratio=0.321016 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=9 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.501339713e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none + [2026-09-21 08:32:49] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 210.9 | throughput per GPU (TFLOP/s/GPU): 152.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.082822E+01 | loss scale: 1.0 | grad norm: 20.631 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44606 payload_rows_remote=39080 dup_rows=20930 dup_rows_remote=18280 dedup_ratio=0.319366 router_tokens_per_expert=[485, 513, 509, 516, 513, 501, 536, 500, 541, 517, 514, 530, 492, 547, 505, 498, 498, 525, 504, 529, 542, 532, 518, 538, 499, 552, 529, 504, 492, 532, 486, 494, 528, 505, 519, 532, 498, 475, 496, 509, 525, 518, 474, 500, 490, 449, 525, 519, 536, 495, 546, 493, 492, 519, 523, 512, 546, 527, 510, 529, 519, 484, 496, 488] payload_rows_per_dst=[5586, 5612, 5646, 5526, 5546, 5534, 5634, 5522] dup_rows_per_dst=[2560, 2676, 2726, 2650, 2578, 2466, 2598, 2676] combine_rows_per_src=[5586, 5612, 5646, 5526, 5546, 5534, 5634, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44552 payload_rows_remote=38914 dup_rows=20984 dup_rows_remote=18306 dedup_ratio=0.320190 router_tokens_per_expert=[532, 533, 511, 525, 515, 509, 493, 485, 535, 524, 506, 496, 509, 542, 540, 550, 476, 523, 480, 540, 523, 497, 497, 522, 472, 522, 515, 498, 504, 532, 519, 521, 495, 535, 514, 539, 488, 494, 487, 489, 480, 544, 525, 516, 502, 495, 525, 509, 552, 527, 540, 524, 498, 478, 534, 505, 505, 508, 510, 479, 506, 502, 496, 521] payload_rows_per_dst=[5566, 5626, 5582, 5564, 5442, 5600, 5638, 5534] dup_rows_per_dst=[2640, 2736, 2534, 2602, 2640, 2634, 2678, 2520] combine_rows_per_src=[5566, 5626, 5582, 5564, 5442, 5600, 5638, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=38894 dup_rows=20976 dup_rows_remote=18304 dedup_ratio=0.320068 router_tokens_per_expert=[568, 518, 538, 521, 518, 503, 483, 476, 532, 474, 501, 510, 518, 524, 545, 521, 518, 545, 510, 493, 521, 541, 544, 497, 497, 505, 531, 541, 521, 526, 487, 490, 496, 513, 467, 528, 530, 490, 523, 548, 480, 491, 514, 508, 499, 549, 493, 513, 503, 517, 493, 475, 521, 512, 511, 509, 556, 514, 494, 489, 506, 500, 488, 521] payload_rows_per_dst=[5606, 5592, 5666, 5586, 5606, 5494, 5492, 5518] dup_rows_per_dst=[2644, 2658, 2672, 2610, 2584, 2600, 2590, 2618] combine_rows_per_src=[5606, 5592, 5666, 5586, 5606, 5494, 5492, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38860 dup_rows=21092 dup_rows_remote=18372 dedup_ratio=0.321838 router_tokens_per_expert=[474, 507, 475, 541, 483, 508, 529, 515, 527, 489, 487, 522, 475, 532, 567, 500, 483, 517, 507, 482, 543, 506, 527, 538, 508, 542, 513, 533, 466, 517, 538, 528, 502, 503, 555, 532, 506, 524, 527, 503, 506, 500, 521, 528, 504, 482, 520, 496, 480, 513, 502, 524, 528, 508, 492, 528, 526, 545, 495, 519, 516, 494, 512, 498] payload_rows_per_dst=[5522, 5528, 5540, 5658, 5584, 5534, 5534, 5544] dup_rows_per_dst=[2542, 2670, 2666, 2632, 2720, 2580, 2616, 2666] combine_rows_per_src=[5522, 5528, 5540, 5658, 5584, 5534, 5534, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44674 payload_rows_remote=39094 dup_rows=20862 dup_rows_remote=18284 dedup_ratio=0.318329 router_tokens_per_expert=[516, 501, 517, 475, 508, 499, 562, 499, 500, 508, 567, 507, 515, 479, 500, 503, 485, 526, 536, 522, 508, 507, 512, 522, 537, 534, 516, 505, 534, 512, 502, 525, 522, 542, 533, 495, 542, 455, 498, 512, 515, 488, 495, 507, 537, 485, 490, 520, 493, 523, 520, 478, 534, 513, 515, 509, 529, 504, 513, 500, 498, 544, 497, 523] payload_rows_per_dst=[5580, 5580, 5566, 5638, 5592, 5590, 5494, 5634] dup_rows_per_dst=[2574, 2578, 2670, 2692, 2606, 2484, 2676, 2582] combine_rows_per_src=[5580, 5580, 5566, 5638, 5592, 5590, 5494, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38922 dup_rows=21082 dup_rows_remote=18596 dedup_ratio=0.321686 router_tokens_per_expert=[507, 525, 529, 521, 518, 495, 526, 513, 485, 536, 542, 509, 545, 484, 478, 536, 530, 495, 553, 478, 515, 511, 477, 500, 526, 491, 555, 512, 523, 502, 560, 521, 520, 463, 505, 480, 539, 519, 534, 486, 498, 514, 522, 508, 471, 524, 476, 496, 514, 495, 513, 489, 516, 538, 490, 504, 507, 515, 493, 501, 585, 537, 528, 490] payload_rows_per_dst=[5596, 5482, 5540, 5710, 5452, 5532, 5522, 5620] dup_rows_per_dst=[2672, 2748, 2578, 2670, 2640, 2486, 2596, 2692] combine_rows_per_src=[5596, 5482, 5540, 5710, 5452, 5532, 5522, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38864 dup_rows=21118 dup_rows_remote=18520 dedup_ratio=0.322235 router_tokens_per_expert=[507, 515, 511, 543, 554, 494, 517, 468, 547, 526, 504, 500, 543, 508, 521, 525, 511, 529, 526, 517, 553, 530, 504, 479, 553, 506, 478, 496, 534, 479, 523, 492, 478, 458, 508, 504, 521, 535, 497, 516, 475, 530, 509, 483, 491, 493, 522, 535, 554, 488, 522, 524, 505, 486, 530, 550, 495, 500, 460, 502, 526, 544, 507, 527] payload_rows_per_dst=[5574, 5150, 5648, 5346, 5674, 5938, 5534, 5554] dup_rows_per_dst=[2644, 2156, 2650, 2386, 2750, 3180, 2754, 2598] combine_rows_per_src=[5574, 5150, 5648, 5346, 5674, 5938, 5534, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160071680 dispatch_meta_bytes=229440 combine_payload_bytes=160071680 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160301120 ingress_bytes=210403328 link_bytes=210403328 token_bytes=320372800 weight_movement_bytes=50331648 total_nvlink_bytes=370704448 weight_to_token_ratio=0.157103 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600926218e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159391744 dispatch_meta_bytes=228880 combine_payload_bytes=159391744 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159620624 ingress_bytes=209723392 link_bytes=209723392 token_bytes=319012368 weight_movement_bytes=50331648 total_nvlink_bytes=369344016 weight_to_token_ratio=0.157773 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595752691e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=228792 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159538616 ingress_bytes=209641472 link_bytes=209641472 token_bytes=318848440 weight_movement_bytes=50331648 total_nvlink_bytes=369180088 weight_to_token_ratio=0.157854 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595129375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159170560 dispatch_meta_bytes=228928 combine_payload_bytes=159170560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159399488 ingress_bytes=184336384 link_bytes=184336384 token_bytes=318570048 weight_movement_bytes=25165824 total_nvlink_bytes=343735872 weight_to_token_ratio=0.078996 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402586893e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160129024 dispatch_meta_bytes=229512 combine_payload_bytes=160129024 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160358536 ingress_bytes=210460672 link_bytes=210460672 token_bytes=320487560 weight_movement_bytes=50331648 total_nvlink_bytes=370819208 weight_to_token_ratio=0.157047 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601362540e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=230072 combine_payload_bytes=159424512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159654584 ingress_bytes=184590336 link_bytes=184590336 token_bytes=319079096 weight_movement_bytes=25165824 total_nvlink_bytes=344244920 weight_to_token_ratio=0.078870 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404519175e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=39020 dup_rows=20990 dup_rows_remote=18286 dedup_ratio=0.320282 router_tokens_per_expert=[550, 482, 508, 508, 528, 484, 500, 523, 490, 540, 501, 531, 526, 511, 521, 520, 546, 527, 485, 505, 531, 503, 504, 479, 479, 513, 484, 528, 526, 485, 525, 543, 543, 503, 483, 512, 540, 513, 465, 502, 528, 475, 536, 460, 473, 485, 493, 492, 526, 510, 544, 520, 512, 531, 482, 543, 526, 526, 541, 533, 548, 516, 497, 524] payload_rows_per_dst=[5526, 5776, 5438, 5596, 5518, 5494, 5570, 5628] dup_rows_per_dst=[2704, 2968, 2414, 2670, 2604, 2390, 2610, 2630] combine_rows_per_src=[5526, 5776, 5438, 5596, 5518, 5494, 5570, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159186944 dispatch_meta_bytes=229536 combine_payload_bytes=159186944 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159416480 ingress_bytes=234684416 link_bytes=234684416 token_bytes=318603424 weight_movement_bytes=75497472 total_nvlink_bytes=394100896 weight_to_token_ratio=0.236964 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785677243e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44280 payload_rows_remote=38844 dup_rows=21256 dup_rows_remote=18720 dedup_ratio=0.324341 router_tokens_per_expert=[513, 535, 540, 498, 503, 504, 499, 538, 535, 503, 512, 509, 476, 520, 520, 517, 500, 528, 494, 499, 555, 491, 497, 522, 496, 486, 507, 482, 480, 514, 542, 479, 526, 518, 488, 517, 472, 539, 513, 529, 479, 499, 535, 490, 486, 530, 494, 475, 518, 513, 530, 533, 535, 512, 525, 495, 517, 565, 518, 537, 491, 539, 508, 548] payload_rows_per_dst=[5574, 5488, 5530, 5436, 5496, 5506, 5622, 5628] dup_rows_per_dst=[2686, 2696, 2642, 2536, 2708, 2470, 2700, 2818] combine_rows_per_src=[5574, 5488, 5530, 5436, 5496, 5506, 5622, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159825920 dispatch_meta_bytes=229224 combine_payload_bytes=159825920 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160055144 ingress_bytes=184991744 link_bytes=184991744 token_bytes=319881064 weight_movement_bytes=25165824 total_nvlink_bytes=345046888 weight_to_token_ratio=0.078672 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407573426e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44358 payload_rows_remote=38724 dup_rows=21178 dup_rows_remote=18466 dedup_ratio=0.323151 router_tokens_per_expert=[531, 466, 506, 522, 515, 486, 561, 505, 525, 548, 534, 537, 516, 539, 456, 538, 525, 473, 484, 524, 470, 534, 486, 517, 500, 511, 479, 500, 476, 547, 541, 504, 549, 470, 495, 515, 504, 518, 482, 497, 542, 544, 474, 480, 514, 528, 502, 555, 495, 524, 522, 517, 520, 523, 514, 558, 513, 515, 506, 523, 514, 511, 466, 522] payload_rows_per_dst=[5596, 5678, 5446, 5496, 5434, 5530, 5634, 5544] dup_rows_per_dst=[2588, 2708, 2580, 2620, 2626, 2748, 2712, 2596] combine_rows_per_src=[5596, 5678, 5446, 5496, 5434, 5530, 5634, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44618 payload_rows_remote=39028 dup_rows=20918 dup_rows_remote=18306 dedup_ratio=0.319183 router_tokens_per_expert=[539, 511, 532, 498, 500, 498, 536, 510, 503, 488, 504, 543, 510, 506, 514, 511, 510, 504, 538, 546, 509, 543, 533, 495, 489, 516, 499, 504, 520, 525, 527, 517, 520, 528, 539, 483, 470, 546, 494, 521, 489, 482, 490, 506, 500, 516, 541, 509, 524, 513, 518, 524, 472, 535, 482, 495, 568, 487, 526, 522, 480, 490, 491, 529] payload_rows_per_dst=[5592, 5518, 5664, 5596, 5590, 5618, 5520, 5520] dup_rows_per_dst=[2656, 2640, 2692, 2598, 2612, 2448, 2606, 2666] combine_rows_per_src=[5592, 5518, 5664, 5596, 5590, 5618, 5520, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=39082 dup_rows=20968 dup_rows_remote=18340 dedup_ratio=0.319946 router_tokens_per_expert=[567, 501, 476, 523, 503, 517, 497, 526, 516, 515, 524, 532, 494, 563, 548, 498, 482, 480, 503, 540, 512, 514, 518, 508, 515, 499, 493, 505, 531, 498, 522, 490, 492, 513, 481, 520, 503, 526, 533, 518, 476, 517, 508, 493, 558, 531, 533, 539, 513, 518, 522, 506, 526, 503, 531, 509, 508, 506, 461, 526, 466, 514, 506, 502] payload_rows_per_dst=[5616, 5626, 5486, 5570, 5598, 5582, 5608, 5482] dup_rows_per_dst=[2604, 2754, 2628, 2536, 2574, 2728, 2648, 2496] combine_rows_per_src=[5616, 5626, 5486, 5570, 5598, 5582, 5608, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44586 payload_rows_remote=39008 dup_rows=20950 dup_rows_remote=18364 dedup_ratio=0.319672 router_tokens_per_expert=[489, 503, 500, 519, 535, 525, 508, 530, 523, 497, 510, 513, 507, 497, 522, 513, 509, 529, 508, 564, 535, 510, 504, 498, 497, 507, 519, 525, 497, 532, 508, 511, 468, 504, 539, 501, 499, 470, 497, 519, 492, 500, 536, 508, 481, 497, 527, 500, 516, 493, 488, 506, 508, 528, 494, 522, 533, 493, 551, 530, 515, 541, 538, 530] payload_rows_per_dst=[5578, 5578, 5602, 5608, 5452, 5534, 5568, 5666] dup_rows_per_dst=[2640, 2586, 2712, 2584, 2542, 2548, 2542, 2796] combine_rows_per_src=[5578, 5578, 5602, 5608, 5452, 5534, 5568, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38828 dup_rows=21114 dup_rows_remote=18400 dedup_ratio=0.322174 router_tokens_per_expert=[503, 529, 496, 539, 496, 516, 552, 522, 479, 484, 504, 530, 506, 491, 499, 484, 513, 542, 497, 516, 543, 480, 541, 513, 501, 521, 547, 528, 489, 495, 498, 502, 517, 496, 506, 504, 497, 530, 491, 531, 551, 542, 499, 533, 512, 537, 499, 481, 506, 495, 459, 454, 523, 508, 498, 498, 529, 511, 499, 545, 555, 529, 532, 545] payload_rows_per_dst=[5596, 5482, 5622, 5522, 5550, 5594, 5388, 5668] dup_rows_per_dst=[2710, 2472, 2668, 2640, 2594, 2714, 2494, 2822] combine_rows_per_src=[5596, 5482, 5622, 5522, 5550, 5594, 5388, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=38852 dup_rows=21086 dup_rows_remote=18350 dedup_ratio=0.321747 router_tokens_per_expert=[526, 514, 528, 506, 503, 485, 532, 494, 507, 495, 501, 507, 490, 539, 512, 499, 539, 546, 512, 501, 491, 516, 536, 502, 506, 525, 498, 535, 512, 470, 505, 502, 526, 521, 527, 505, 491, 522, 512, 520, 514, 495, 498, 498, 526, 471, 508, 478, 500, 560, 541, 542, 509, 493, 470, 543, 504, 500, 491, 530, 551, 542, 502, 544] payload_rows_per_dst=[5406, 5534, 5652, 5456, 5638, 5564, 5602, 5598] dup_rows_per_dst=[2472, 2566, 2634, 2650, 2610, 2710, 2708, 2736] combine_rows_per_src=[5406, 5534, 5652, 5456, 5638, 5564, 5602, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=230256 combine_payload_bytes=159105024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159335280 ingress_bytes=184270848 link_bytes=184270848 token_bytes=318440304 weight_movement_bytes=25165824 total_nvlink_bytes=343606128 weight_to_token_ratio=0.079028 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402088240e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158613504 dispatch_meta_bytes=228760 combine_payload_bytes=158613504 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158842264 ingress_bytes=208945152 link_bytes=208945152 token_bytes=317455768 weight_movement_bytes=50331648 total_nvlink_bytes=367787416 weight_to_token_ratio=0.158547 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589831184e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159858688 dispatch_meta_bytes=229336 combine_payload_bytes=159858688 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160088024 ingress_bytes=185024512 link_bytes=185024512 token_bytes=319946712 weight_movement_bytes=25165824 total_nvlink_bytes=345112536 weight_to_token_ratio=0.078656 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407822752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160079872 dispatch_meta_bytes=229688 combine_payload_bytes=160079872 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160309560 ingress_bytes=210411520 link_bytes=210411520 token_bytes=320389432 weight_movement_bytes=50331648 total_nvlink_bytes=370721080 weight_to_token_ratio=0.157095 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600988550e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159776768 dispatch_meta_bytes=229488 combine_payload_bytes=159776768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160006256 ingress_bytes=184942592 link_bytes=184942592 token_bytes=319783024 weight_movement_bytes=25165824 total_nvlink_bytes=344948848 weight_to_token_ratio=0.078697 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407199436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159039488 dispatch_meta_bytes=228912 combine_payload_bytes=159039488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159268400 ingress_bytes=184205312 link_bytes=184205312 token_bytes=318307888 weight_movement_bytes=25165824 total_nvlink_bytes=343473712 weight_to_token_ratio=0.079061 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401589587e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159137792 dispatch_meta_bytes=228808 combine_payload_bytes=159137792 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159366600 ingress_bytes=259801088 link_bytes=259801088 token_bytes=318504392 weight_movement_bytes=100663296 total_nvlink_bytes=419167688 weight_to_token_ratio=0.316050 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.976786096e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44582 payload_rows_remote=39018 dup_rows=20954 dup_rows_remote=18296 dedup_ratio=0.319733 router_tokens_per_expert=[510, 494, 521, 504, 471, 527, 516, 496, 484, 496, 506, 512, 493, 531, 503, 537, 526, 479, 529, 516, 511, 486, 481, 526, 522, 506, 529, 499, 500, 524, 527, 530, 515, 494, 510, 510, 513, 486, 543, 525, 536, 472, 512, 518, 515, 512, 521, 535, 541, 477, 531, 518, 521, 513, 479, 519, 503, 529, 541, 496, 531, 526, 509, 525] payload_rows_per_dst=[5564, 5556, 5486, 5740, 5710, 5602, 5622, 5302] dup_rows_per_dst=[2658, 2654, 2492, 2948, 2802, 2640, 2562, 2198] combine_rows_per_src=[5564, 5556, 5486, 5740, 5710, 5602, 5622, 5302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44446 payload_rows_remote=38836 dup_rows=21090 dup_rows_remote=18380 dedup_ratio=0.321808 router_tokens_per_expert=[520, 530, 494, 512, 471, 552, 523, 493, 537, 524, 499, 512, 504, 551, 491, 534, 489, 537, 463, 504, 500, 477, 507, 518, 548, 461, 525, 534, 495, 492, 577, 528, 487, 489, 520, 496, 508, 536, 512, 510, 484, 474, 537, 458, 497, 526, 514, 516, 486, 531, 519, 524, 512, 539, 517, 540, 534, 493, 511, 529, 534, 531, 506, 496] payload_rows_per_dst=[5558, 5654, 5474, 5610, 5498, 5448, 5638, 5566] dup_rows_per_dst=[2632, 2650, 2516, 2710, 2618, 2564, 2698, 2702] combine_rows_per_src=[5558, 5654, 5474, 5610, 5498, 5448, 5638, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=229256 combine_payload_bytes=159817728 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160046984 ingress_bytes=210149376 link_bytes=210149376 token_bytes=319864712 weight_movement_bytes=50331648 total_nvlink_bytes=370196360 weight_to_token_ratio=0.157353 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598993937e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44610 payload_rows_remote=38890 dup_rows=20926 dup_rows_remote=18196 dedup_ratio=0.319305 router_tokens_per_expert=[462, 511, 539, 523, 461, 524, 516, 522, 518, 543, 536, 488, 500, 522, 508, 514, 486, 470, 545, 492, 489, 526, 525, 512, 488, 512, 464, 489, 490, 504, 502, 511, 512, 493, 498, 539, 497, 528, 534, 519, 493, 504, 545, 527, 509, 480, 549, 482, 544, 549, 527, 517, 525, 531, 532, 500, 516, 544, 528, 526, 521, 524, 478, 505] payload_rows_per_dst=[5508, 5650, 5544, 5486, 5606, 5516, 5720, 5580] dup_rows_per_dst=[2608, 2608, 2546, 2434, 2634, 2662, 2730, 2704] combine_rows_per_src=[5508, 5650, 5544, 5486, 5606, 5516, 5720, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44330 payload_rows_remote=38738 dup_rows=21206 dup_rows_remote=18578 dedup_ratio=0.323578 router_tokens_per_expert=[509, 534, 482, 504, 515, 474, 514, 542, 506, 525, 543, 490, 503, 502, 491, 501, 536, 527, 491, 492, 507, 539, 517, 501, 496, 503, 500, 532, 494, 518, 492, 535, 521, 497, 524, 539, 517, 515, 470, 492, 550, 532, 549, 552, 490, 515, 486, 512, 516, 499, 550, 521, 489, 548, 492, 523, 501, 500, 511, 453, 527, 532, 519, 511] payload_rows_per_dst=[5584, 5538, 5592, 5470, 5524, 5614, 5528, 5480] dup_rows_per_dst=[2564, 2584, 2628, 2670, 2626, 2758, 2748, 2628] combine_rows_per_src=[5584, 5538, 5592, 5470, 5524, 5614, 5528, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44376 payload_rows_remote=38900 dup_rows=21160 dup_rows_remote=18540 dedup_ratio=0.322876 router_tokens_per_expert=[514, 524, 526, 526, 508, 496, 505, 499, 500, 524, 527, 492, 494, 500, 517, 523, 498, 515, 508, 473, 507, 511, 534, 530, 542, 541, 545, 522, 476, 507, 531, 531, 479, 521, 514, 508, 493, 525, 494, 514, 520, 500, 522, 503, 526, 570, 518, 490, 509, 514, 487, 510, 467, 506, 482, 529, 515, 536, 488, 502, 540, 528, 510, 502] payload_rows_per_dst=[5556, 5510, 5540, 5686, 5476, 5592, 5462, 5554] dup_rows_per_dst=[2640, 2644, 2612, 2704, 2620, 2706, 2546, 2688] combine_rows_per_src=[5556, 5510, 5540, 5686, 5476, 5592, 5462, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44184 payload_rows_remote=38594 dup_rows=21352 dup_rows_remote=18702 dedup_ratio=0.325806 router_tokens_per_expert=[474, 550, 546, 547, 501, 516, 521, 485, 499, 537, 502, 505, 508, 518, 522, 529, 499, 556, 488, 512, 539, 515, 488, 511, 495, 517, 479, 547, 507, 522, 521, 512, 504, 503, 499, 529, 495, 514, 485, 476, 539, 549, 492, 551, 524, 471, 523, 499, 493, 525, 506, 532, 487, 507, 537, 526, 503, 483, 502, 534, 484, 492, 502, 534] payload_rows_per_dst=[5576, 5590, 5566, 5496, 5468, 5518, 5524, 5446] dup_rows_per_dst=[2704, 2650, 2650, 2704, 2542, 2778, 2702, 2622] combine_rows_per_src=[5576, 5590, 5566, 5496, 5468, 5518, 5524, 5446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44744 payload_rows_remote=39094 dup_rows=20792 dup_rows_remote=18132 dedup_ratio=0.317261 router_tokens_per_expert=[527, 475, 481, 525, 519, 501, 515, 533, 516, 555, 490, 508, 511, 527, 506, 499, 543, 503, 522, 488, 513, 500, 514, 513, 502, 517, 470, 509, 490, 504, 506, 524, 527, 505, 474, 543, 495, 503, 504, 571, 509, 542, 508, 545, 518, 482, 522, 529, 492, 533, 510, 510, 501, 511, 545, 504, 531, 512, 522, 464, 517, 531, 510, 492] payload_rows_per_dst=[5514, 5606, 5648, 5558, 5618, 5650, 5556, 5594] dup_rows_per_dst=[2638, 2618, 2544, 2486, 2626, 2660, 2656, 2564] combine_rows_per_src=[5514, 5606, 5648, 5558, 5618, 5650, 5556, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38860 dup_rows=21046 dup_rows_remote=18330 dedup_ratio=0.321136 router_tokens_per_expert=[522, 507, 500, 504, 525, 538, 504, 508, 492, 549, 483, 450, 494, 530, 514, 519, 518, 505, 544, 495, 509, 458, 500, 500, 517, 505, 507, 528, 551, 528, 514, 557, 545, 500, 507, 496, 512, 479, 490, 507, 514, 535, 505, 520, 536, 520, 514, 504, 511, 520, 452, 508, 535, 503, 518, 556, 522, 502, 500, 515, 500, 535, 507, 525] payload_rows_per_dst=[5570, 5472, 5492, 5678, 5540, 5570, 5538, 5630] dup_rows_per_dst=[2646, 2526, 2566, 2736, 2532, 2740, 2584, 2716] combine_rows_per_src=[5570, 5472, 5492, 5678, 5540, 5570, 5538, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159072256 dispatch_meta_bytes=228864 combine_payload_bytes=159072256 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159301120 ingress_bytes=184238080 link_bytes=184238080 token_bytes=318373376 weight_movement_bytes=25165824 total_nvlink_bytes=343539200 weight_to_token_ratio=0.079045 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401838913e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=228344 combine_payload_bytes=159293440 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159521784 ingress_bytes=234790912 link_bytes=234790912 token_bytes=318815224 weight_movement_bytes=75497472 total_nvlink_bytes=394312696 weight_to_token_ratio=0.236806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786487554e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158670848 dispatch_meta_bytes=229264 combine_payload_bytes=158670848 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158900112 ingress_bytes=183836672 link_bytes=183836672 token_bytes=317570960 weight_movement_bytes=25165824 total_nvlink_bytes=342736784 weight_to_token_ratio=0.079245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398784662e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=229760 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159564160 ingress_bytes=184500224 link_bytes=184500224 token_bytes=318898560 weight_movement_bytes=25165824 total_nvlink_bytes=344064384 weight_to_token_ratio=0.078915 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403833526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158081024 dispatch_meta_bytes=229184 combine_payload_bytes=158081024 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=158310208 ingress_bytes=233578496 link_bytes=233578496 token_bytes=316391232 weight_movement_bytes=75497472 total_nvlink_bytes=391888704 weight_to_token_ratio=0.238621 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.777262470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=160129024 dispatch_meta_bytes=228904 combine_payload_bytes=160129024 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160357928 ingress_bytes=235626496 link_bytes=235626496 token_bytes=320486952 weight_movement_bytes=75497472 total_nvlink_bytes=395984424 weight_to_token_ratio=0.235571 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792845383e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159170560 dispatch_meta_bytes=228760 combine_payload_bytes=159170560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159399320 ingress_bytes=184336384 link_bytes=184336384 token_bytes=318569880 weight_movement_bytes=25165824 total_nvlink_bytes=343735704 weight_to_token_ratio=0.078996 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402586893e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=38978 dup_rows=20976 dup_rows_remote=18320 dedup_ratio=0.320068 router_tokens_per_expert=[517, 515, 502, 518, 511, 482, 483, 508, 553, 513, 536, 518, 543, 510, 502, 526, 549, 532, 524, 555, 493, 471, 529, 503, 512, 499, 508, 531, 511, 494, 482, 501, 480, 543, 527, 480, 544, 543, 496, 485, 483, 510, 538, 514, 495, 513, 551, 498, 510, 513, 531, 513, 529, 496, 541, 473, 506, 515, 477, 524, 539, 495, 490, 485] payload_rows_per_dst=[5582, 5582, 5740, 5576, 5724, 5400, 5426, 5530] dup_rows_per_dst=[2656, 2654, 2878, 2532, 2884, 2360, 2480, 2532] combine_rows_per_src=[5582, 5582, 5740, 5576, 5724, 5400, 5426, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=38956 dup_rows=20978 dup_rows_remote=18470 dedup_ratio=0.320099 router_tokens_per_expert=[501, 527, 505, 533, 495, 482, 524, 504, 507, 509, 504, 489, 522, 482, 505, 553, 500, 508, 521, 550, 496, 529, 549, 506, 494, 543, 451, 509, 539, 512, 513, 494, 519, 520, 537, 532, 533, 509, 568, 500, 489, 518, 478, 508, 515, 486, 505, 510, 508, 548, 519, 529, 505, 480, 505, 497, 530, 500, 503, 510, 513, 519, 518, 501] payload_rows_per_dst=[5520, 5544, 5640, 5602, 5696, 5444, 5570, 5542] dup_rows_per_dst=[2622, 2598, 2678, 2508, 2740, 2574, 2612, 2646] combine_rows_per_src=[5520, 5544, 5640, 5602, 5696, 5444, 5570, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159653888 dispatch_meta_bytes=229192 combine_payload_bytes=159653888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159883080 ingress_bytes=184819712 link_bytes=184819712 token_bytes=319536968 weight_movement_bytes=25165824 total_nvlink_bytes=344702792 weight_to_token_ratio=0.078757 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406264461e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44512 payload_rows_remote=38982 dup_rows=21024 dup_rows_remote=18322 dedup_ratio=0.320801 router_tokens_per_expert=[513, 552, 521, 469, 515, 508, 515, 472, 496, 507, 503, 496, 529, 526, 485, 523, 503, 472, 514, 509, 542, 495, 489, 551, 503, 539, 472, 564, 580, 519, 513, 543, 492, 486, 518, 531, 501, 518, 496, 526, 494, 538, 499, 487, 469, 558, 558, 481, 504, 520, 498, 505, 534, 534, 522, 499, 500, 530, 510, 498, 501, 523, 493, 507] payload_rows_per_dst=[5576, 5546, 5520, 5734, 5572, 5532, 5530, 5502] dup_rows_per_dst=[2554, 2584, 2630, 2732, 2564, 2636, 2702, 2622] combine_rows_per_src=[5576, 5546, 5520, 5734, 5572, 5532, 5530, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=39000 dup_rows=21096 dup_rows_remote=18516 dedup_ratio=0.321899 router_tokens_per_expert=[537, 498, 498, 517, 554, 476, 495, 504, 528, 505, 482, 557, 518, 516, 500, 525, 497, 559, 526, 509, 516, 515, 472, 510, 522, 514, 491, 509, 558, 511, 534, 513, 500, 516, 504, 479, 509, 504, 480, 518, 486, 545, 501, 547, 477, 502, 501, 498, 503, 511, 544, 532, 514, 515, 513, 509, 491, 520, 535, 521, 467, 492, 527, 541] payload_rows_per_dst=[5576, 5574, 5582, 5588, 5440, 5502, 5604, 5574] dup_rows_per_dst=[2582, 2688, 2626, 2716, 2580, 2612, 2678, 2614] combine_rows_per_src=[5576, 5574, 5582, 5588, 5440, 5502, 5604, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44498 payload_rows_remote=38984 dup_rows=21038 dup_rows_remote=18486 dedup_ratio=0.321014 router_tokens_per_expert=[541, 485, 544, 559, 507, 524, 552, 520, 529, 490, 487, 488, 527, 489, 497, 491, 467, 518, 500, 486, 545, 515, 496, 506, 492, 514, 539, 496, 519, 490, 520, 510, 524, 539, 497, 509, 514, 507, 519, 471, 488, 538, 559, 497, 511, 547, 522, 494, 523, 505, 520, 538, 504, 532, 518, 514, 503, 495, 511, 504, 491, 514, 512, 505] payload_rows_per_dst=[5676, 5498, 5514, 5602, 5516, 5662, 5576, 5454] dup_rows_per_dst=[2788, 2498, 2552, 2558, 2644, 2650, 2732, 2616] combine_rows_per_src=[5676, 5498, 5514, 5602, 5516, 5662, 5576, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44580 payload_rows_remote=39052 dup_rows=20956 dup_rows_remote=18442 dedup_ratio=0.319763 router_tokens_per_expert=[513, 485, 525, 515, 519, 508, 484, 519, 514, 492, 520, 470, 507, 488, 533, 497, 504, 502, 545, 524, 480, 518, 523, 522, 582, 514, 526, 511, 501, 495, 536, 548, 514, 491, 551, 522, 517, 490, 516, 495, 510, 480, 518, 506, 528, 535, 520, 501, 472, 509, 484, 520, 516, 491, 511, 516, 516, 523, 513, 522, 514, 512, 488, 547] payload_rows_per_dst=[5524, 5528, 5618, 5660, 5578, 5562, 5482, 5628] dup_rows_per_dst=[2612, 2514, 2618, 2766, 2614, 2634, 2556, 2642] combine_rows_per_src=[5524, 5528, 5618, 5660, 5578, 5562, 5482, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=38882 dup_rows=20968 dup_rows_remote=18358 dedup_ratio=0.319946 router_tokens_per_expert=[518, 500, 548, 542, 497, 466, 514, 507, 539, 521, 535, 471, 507, 509, 505, 541, 517, 523, 493, 523, 507, 503, 531, 501, 492, 529, 531, 488, 465, 521, 483, 517, 487, 503, 547, 527, 494, 523, 511, 536, 508, 498, 521, 538, 519, 479, 501, 484, 560, 514, 509, 472, 507, 483, 537, 520, 522, 486, 498, 477, 560, 526, 525, 552] payload_rows_per_dst=[5588, 5578, 5612, 5494, 5540, 5560, 5510, 5686] dup_rows_per_dst=[2596, 2678, 2548, 2590, 2716, 2536, 2694, 2610] combine_rows_per_src=[5588, 5578, 5612, 5494, 5540, 5560, 5510, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44542 payload_rows_remote=38914 dup_rows=20994 dup_rows_remote=18278 dedup_ratio=0.320343 router_tokens_per_expert=[520, 476, 536, 497, 525, 492, 482, 529, 513, 504, 526, 492, 498, 490, 503, 496, 525, 509, 549, 513, 528, 508, 564, 495, 505, 511, 526, 518, 530, 496, 512, 496, 500, 510, 483, 501, 485, 525, 513, 481, 517, 511, 533, 506, 489, 523, 546, 547, 531, 504, 546, 494, 510, 489, 478, 546, 528, 524, 520, 522, 532, 515, 500, 495] payload_rows_per_dst=[5544, 5524, 5632, 5556, 5470, 5628, 5580, 5608] dup_rows_per_dst=[2570, 2520, 2750, 2632, 2526, 2716, 2616, 2664] combine_rows_per_src=[5544, 5524, 5632, 5556, 5470, 5628, 5580, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=229704 combine_payload_bytes=159563776 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159793480 ingress_bytes=209895424 link_bytes=209895424 token_bytes=319357256 weight_movement_bytes=50331648 total_nvlink_bytes=369688904 weight_to_token_ratio=0.157603 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597061656e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229216 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159899488 ingress_bytes=210001920 link_bytes=210001920 token_bytes=319569760 weight_movement_bytes=50331648 total_nvlink_bytes=369901408 weight_to_token_ratio=0.157498 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597871967e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159744000 dispatch_meta_bytes=230064 combine_payload_bytes=159744000 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159974064 ingress_bytes=184909824 link_bytes=184909824 token_bytes=319718064 weight_movement_bytes=25165824 total_nvlink_bytes=344883888 weight_to_token_ratio=0.078713 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406950109e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159678464 dispatch_meta_bytes=229880 combine_payload_bytes=159678464 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159908344 ingress_bytes=235175936 link_bytes=235175936 token_bytes=319586808 weight_movement_bytes=75497472 total_nvlink_bytes=395084280 weight_to_token_ratio=0.236235 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789417142e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159956992 dispatch_meta_bytes=229976 combine_payload_bytes=159956992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160186968 ingress_bytes=185122816 link_bytes=185122816 token_bytes=320143960 weight_movement_bytes=25165824 total_nvlink_bytes=345309784 weight_to_token_ratio=0.078608 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408570732e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44378 payload_rows_remote=38856 dup_rows=21158 dup_rows_remote=18472 dedup_ratio=0.322845 router_tokens_per_expert=[501, 457, 536, 513, 512, 493, 533, 495, 492, 477, 549, 524, 511, 519, 510, 517, 518, 524, 505, 487, 533, 512, 512, 481, 505, 521, 488, 540, 543, 504, 516, 515, 491, 526, 504, 488, 504, 504, 493, 547, 504, 513, 531, 507, 510, 495, 510, 516, 520, 512, 507, 499, 539, 557, 495, 516, 533, 530, 537, 491, 524, 514, 486, 522] payload_rows_per_dst=[5522, 5746, 5468, 5336, 5616, 5612, 5476, 5602] dup_rows_per_dst=[2686, 2918, 2548, 2462, 2724, 2676, 2618, 2526] combine_rows_per_src=[5522, 5746, 5468, 5336, 5616, 5612, 5476, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159260672 dispatch_meta_bytes=228960 combine_payload_bytes=159260672 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159489632 ingress_bytes=234758144 link_bytes=234758144 token_bytes=318750304 weight_movement_bytes=75497472 total_nvlink_bytes=394247776 weight_to_token_ratio=0.236855 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786238228e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159391744 dispatch_meta_bytes=228768 combine_payload_bytes=159391744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159620512 ingress_bytes=184557568 link_bytes=184557568 token_bytes=319012256 weight_movement_bytes=25165824 total_nvlink_bytes=344178080 weight_to_token_ratio=0.078887 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404269848e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229312 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159383488 ingress_bytes=184320000 link_bytes=184320000 token_bytes=318537664 weight_movement_bytes=25165824 total_nvlink_bytes=343703488 weight_to_token_ratio=0.079004 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402462230e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38898 dup_rows=21114 dup_rows_remote=18436 dedup_ratio=0.322174 router_tokens_per_expert=[504, 531, 493, 500, 548, 512, 489, 540, 491, 476, 505, 517, 534, 525, 532, 492, 519, 504, 535, 517, 485, 544, 498, 526, 527, 547, 491, 503, 535, 544, 449, 505, 510, 488, 503, 503, 502, 510, 511, 534, 511, 512, 551, 530, 502, 518, 477, 524, 517, 452, 491, 508, 496, 518, 535, 504, 490, 524, 528, 504, 544, 525, 558, 470] payload_rows_per_dst=[5508, 5536, 5600, 5524, 5506, 5614, 5490, 5644] dup_rows_per_dst=[2726, 2608, 2656, 2678, 2616, 2636, 2552, 2642] combine_rows_per_src=[5508, 5536, 5600, 5524, 5506, 5614, 5490, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38886 dup_rows=21146 dup_rows_remote=18484 dedup_ratio=0.322662 router_tokens_per_expert=[496, 527, 517, 461, 518, 506, 517, 507, 511, 497, 550, 505, 525, 515, 495, 516, 469, 518, 538, 494, 532, 490, 518, 504, 481, 490, 545, 531, 529, 502, 551, 510, 519, 504, 524, 514, 500, 501, 502, 549, 499, 539, 534, 558, 539, 469, 491, 532, 479, 540, 526, 494, 541, 500, 491, 512, 524, 519, 491, 516, 495, 491, 513, 497] payload_rows_per_dst=[5550, 5580, 5538, 5518, 5572, 5600, 5504, 5528] dup_rows_per_dst=[2548, 2648, 2588, 2760, 2654, 2722, 2662, 2564] combine_rows_per_src=[5550, 5580, 5538, 5518, 5572, 5600, 5504, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38978 dup_rows=21002 dup_rows_remote=18292 dedup_ratio=0.320465 router_tokens_per_expert=[508, 491, 554, 521, 528, 471, 505, 511, 504, 520, 492, 535, 526, 473, 470, 515, 488, 511, 527, 511, 565, 513, 535, 482, 550, 492, 511, 533, 492, 501, 475, 515, 492, 489, 564, 513, 525, 516, 534, 500, 570, 534, 503, 500, 528, 475, 532, 484, 547, 487, 533, 490, 488, 495, 535, 511, 483, 536, 538, 539, 493, 499, 498, 512] payload_rows_per_dst=[5536, 5586, 5620, 5534, 5556, 5560, 5556, 5586] dup_rows_per_dst=[2642, 2484, 2644, 2604, 2710, 2692, 2616, 2610] combine_rows_per_src=[5536, 5586, 5620, 5534, 5556, 5560, 5556, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44708 payload_rows_remote=39078 dup_rows=20828 dup_rows_remote=18222 dedup_ratio=0.317810 router_tokens_per_expert=[505, 485, 525, 549, 496, 505, 513, 497, 499, 486, 479, 491, 508, 515, 500, 509, 523, 509, 533, 502, 537, 491, 527, 496, 556, 515, 494, 555, 500, 504, 546, 517, 518, 489, 544, 517, 493, 513, 492, 524, 475, 516, 493, 512, 511, 491, 515, 510, 519, 498, 521, 532, 519, 485, 507, 516, 546, 487, 534, 521, 523, 557, 464, 559] payload_rows_per_dst=[5590, 5452, 5630, 5712, 5580, 5494, 5560, 5690] dup_rows_per_dst=[2560, 2522, 2606, 2662, 2600, 2552, 2634, 2692] combine_rows_per_src=[5590, 5452, 5630, 5712, 5580, 5494, 5560, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44426 payload_rows_remote=38828 dup_rows=21110 dup_rows_remote=18468 dedup_ratio=0.322113 router_tokens_per_expert=[500, 510, 514, 485, 513, 498, 516, 510, 498, 561, 536, 464, 509, 514, 535, 503, 526, 501, 488, 493, 547, 499, 517, 562, 475, 501, 543, 481, 534, 539, 499, 531, 527, 476, 486, 535, 504, 534, 491, 526, 488, 513, 496, 506, 500, 544, 534, 492, 517, 505, 528, 547, 537, 490, 532, 533, 501, 518, 540, 502, 463, 513, 496, 492] payload_rows_per_dst=[5506, 5598, 5558, 5540, 5552, 5520, 5698, 5454] dup_rows_per_dst=[2586, 2642, 2708, 2666, 2606, 2626, 2680, 2596] combine_rows_per_src=[5506, 5598, 5558, 5540, 5552, 5520, 5698, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44446 payload_rows_remote=38830 dup_rows=21090 dup_rows_remote=18336 dedup_ratio=0.321808 router_tokens_per_expert=[492, 502, 501, 531, 514, 526, 528, 505, 520, 482, 480, 493, 526, 536, 475, 520, 498, 507, 504, 535, 543, 482, 510, 513, 499, 548, 523, 506, 475, 487, 503, 532, 498, 541, 516, 505, 512, 552, 538, 500, 486, 482, 505, 491, 545, 535, 511, 519, 538, 552, 491, 538, 468, 489, 480, 497, 553, 512, 504, 520, 552, 499, 539, 504] payload_rows_per_dst=[5494, 5504, 5498, 5486, 5674, 5602, 5572, 5616] dup_rows_per_dst=[2704, 2560, 2686, 2514, 2650, 2546, 2676, 2754] combine_rows_per_src=[5494, 5504, 5498, 5486, 5674, 5602, 5572, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38918 dup_rows=21002 dup_rows_remote=18258 dedup_ratio=0.320465 router_tokens_per_expert=[510, 520, 492, 521, 508, 469, 494, 536, 521, 511, 512, 516, 482, 545, 529, 540, 486, 534, 517, 521, 484, 480, 553, 511, 536, 482, 487, 534, 500, 514, 517, 510, 495, 493, 539, 517, 500, 540, 525, 494, 534, 542, 523, 544, 534, 513, 508, 482, 514, 551, 508, 501, 505, 497, 520, 511, 473, 533, 510, 523, 500, 492, 487, 488] payload_rows_per_dst=[5566, 5634, 5504, 5550, 5522, 5616, 5624, 5518] dup_rows_per_dst=[2534, 2678, 2668, 2610, 2684, 2744, 2590, 2494] combine_rows_per_src=[5566, 5634, 5504, 5550, 5522, 5616, 5624, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159326208 dispatch_meta_bytes=229336 combine_payload_bytes=159326208 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159555544 ingress_bytes=209657856 link_bytes=209657856 token_bytes=318881752 weight_movement_bytes=50331648 total_nvlink_bytes=369213400 weight_to_token_ratio=0.157838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595254038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=229480 combine_payload_bytes=159277056 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159506536 ingress_bytes=184442880 link_bytes=184442880 token_bytes=318783592 weight_movement_bytes=25165824 total_nvlink_bytes=343949416 weight_to_token_ratio=0.078943 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403397205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159653888 dispatch_meta_bytes=229080 combine_payload_bytes=159653888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159882968 ingress_bytes=184819712 link_bytes=184819712 token_bytes=319536856 weight_movement_bytes=25165824 total_nvlink_bytes=344702680 weight_to_token_ratio=0.078757 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406264461e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=229200 combine_payload_bytes=160063488 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160292688 ingress_bytes=210395136 link_bytes=210395136 token_bytes=320356176 weight_movement_bytes=50331648 total_nvlink_bytes=370687824 weight_to_token_ratio=0.157112 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600863887e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159039488 dispatch_meta_bytes=229184 combine_payload_bytes=159039488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159268672 ingress_bytes=184205312 link_bytes=184205312 token_bytes=318308160 weight_movement_bytes=25165824 total_nvlink_bytes=343473984 weight_to_token_ratio=0.079061 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401589587e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38820 dup_rows=21092 dup_rows_remote=18230 dedup_ratio=0.321838 router_tokens_per_expert=[493, 506, 518, 498, 488, 510, 528, 508, 548, 487, 519, 525, 510, 535, 510, 499, 529, 540, 455, 513, 507, 539, 517, 534, 488, 503, 555, 500, 495, 508, 528, 512, 507, 487, 494, 489, 501, 521, 481, 521, 507, 561, 530, 509, 499, 551, 514, 530, 511, 526, 489, 534, 513, 517, 488, 483, 486, 515, 544, 483, 520, 544, 512, 496] payload_rows_per_dst=[5624, 5690, 5496, 5560, 5482, 5462, 5554, 5576] dup_rows_per_dst=[2862, 2814, 2536, 2618, 2572, 2550, 2568, 2572] combine_rows_per_src=[5624, 5690, 5496, 5560, 5482, 5462, 5554, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159047680 dispatch_meta_bytes=228664 combine_payload_bytes=159047680 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159276344 ingress_bytes=209379328 link_bytes=209379328 token_bytes=318324024 weight_movement_bytes=50331648 total_nvlink_bytes=368655672 weight_to_token_ratio=0.158115 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593134762e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=228704 combine_payload_bytes=159408128 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159636832 ingress_bytes=209739776 link_bytes=209739776 token_bytes=319044960 weight_movement_bytes=50331648 total_nvlink_bytes=369376608 weight_to_token_ratio=0.157757 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595877354e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=228200 combine_payload_bytes=159006720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159234920 ingress_bytes=184172544 link_bytes=184172544 token_bytes=318241640 weight_movement_bytes=25165824 total_nvlink_bytes=343407464 weight_to_token_ratio=0.079078 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401340260e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44734 payload_rows_remote=39226 dup_rows=20802 dup_rows_remote=18254 dedup_ratio=0.317413 router_tokens_per_expert=[495, 510, 538, 482, 543, 506, 547, 519, 501, 500, 495, 489, 473, 512, 546, 506, 510, 494, 509, 503, 531, 498, 487, 513, 548, 474, 499, 518, 464, 523, 459, 543, 515, 503, 503, 539, 534, 527, 518, 499, 516, 520, 525, 504, 518, 542, 555, 464, 516, 510, 523, 516, 521, 527, 513, 515, 503, 553, 527, 518, 499, 509, 493, 508] payload_rows_per_dst=[5638, 5560, 5504, 5508, 5648, 5656, 5606, 5614] dup_rows_per_dst=[2642, 2484, 2586, 2548, 2628, 2632, 2676, 2606] combine_rows_per_src=[5638, 5560, 5504, 5508, 5648, 5656, 5606, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=39030 dup_rows=20990 dup_rows_remote=18442 dedup_ratio=0.320282 router_tokens_per_expert=[502, 487, 513, 567, 535, 516, 515, 541, 509, 496, 543, 519, 522, 505, 491, 515, 492, 502, 488, 497, 538, 562, 522, 519, 523, 522, 531, 543, 548, 491, 467, 491, 482, 502, 536, 486, 506, 496, 538, 486, 499, 530, 514, 517, 456, 488, 507, 530, 491, 526, 542, 527, 501, 521, 469, 527, 550, 498, 503, 485, 470, 548, 554, 471] payload_rows_per_dst=[5628, 5500, 5552, 5638, 5516, 5568, 5580, 5564] dup_rows_per_dst=[2724, 2700, 2688, 2594, 2548, 2514, 2628, 2594] combine_rows_per_src=[5628, 5500, 5552, 5638, 5516, 5568, 5580, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44532 payload_rows_remote=39044 dup_rows=21004 dup_rows_remote=18438 dedup_ratio=0.320496 router_tokens_per_expert=[550, 521, 516, 546, 495, 527, 497, 489, 511, 492, 515, 480, 487, 514, 538, 481, 540, 526, 552, 505, 534, 542, 504, 516, 539, 535, 518, 514, 530, 524, 507, 541, 480, 461, 483, 481, 502, 480, 516, 533, 504, 509, 547, 502, 524, 510, 499, 493, 467, 470, 490, 512, 545, 505, 531, 507, 521, 475, 477, 515, 539, 541, 539, 524] payload_rows_per_dst=[5594, 5528, 5656, 5678, 5404, 5586, 5488, 5598] dup_rows_per_dst=[2688, 2508, 2782, 2738, 2468, 2590, 2566, 2664] combine_rows_per_src=[5594, 5528, 5656, 5678, 5404, 5586, 5488, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44588 payload_rows_remote=39040 dup_rows=20948 dup_rows_remote=18286 dedup_ratio=0.319641 router_tokens_per_expert=[510, 513, 506, 492, 511, 528, 497, 518, 490, 495, 561, 534, 502, 526, 528, 489, 509, 513, 510, 475, 577, 517, 478, 526, 516, 509, 489, 486, 518, 476, 509, 548, 528, 487, 522, 521, 529, 511, 516, 502, 536, 501, 499, 486, 536, 536, 515, 478, 544, 506, 528, 495, 509, 493, 484, 515, 533, 515, 503, 539, 516, 541, 509, 479] payload_rows_per_dst=[5476, 5614, 5548, 5542, 5590, 5602, 5580, 5636] dup_rows_per_dst=[2674, 2636, 2662, 2560, 2642, 2572, 2568, 2634] combine_rows_per_src=[5476, 5614, 5548, 5542, 5590, 5602, 5580, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44402 payload_rows_remote=38752 dup_rows=21134 dup_rows_remote=18494 dedup_ratio=0.322479 router_tokens_per_expert=[508, 485, 499, 522, 543, 521, 522, 537, 523, 516, 537, 495, 559, 530, 502, 483, 531, 528, 506, 501, 516, 512, 508, 492, 511, 491, 510, 518, 486, 496, 545, 494, 494, 510, 517, 536, 521, 473, 544, 511, 487, 497, 530, 493, 505, 518, 518, 514, 526, 513, 481, 532, 532, 492, 507, 518, 515, 516, 497, 495, 511, 526, 536, 476] payload_rows_per_dst=[5524, 5650, 5584, 5486, 5536, 5512, 5588, 5522] dup_rows_per_dst=[2750, 2640, 2604, 2616, 2676, 2612, 2614, 2622] combine_rows_per_src=[5524, 5650, 5584, 5486, 5536, 5512, 5588, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38978 dup_rows=21072 dup_rows_remote=18618 dedup_ratio=0.321533 router_tokens_per_expert=[508, 510, 531, 514, 534, 547, 531, 523, 503, 486, 514, 486, 512, 498, 499, 499, 535, 533, 517, 507, 528, 510, 509, 518, 496, 488, 481, 504, 526, 530, 533, 502, 504, 512, 505, 511, 465, 548, 492, 524, 525, 536, 514, 520, 482, 522, 540, 524, 520, 495, 490, 513, 520, 575, 543, 506, 517, 475, 497, 486, 468, 516, 474, 537] payload_rows_per_dst=[5364, 5518, 5370, 5672, 5700, 5724, 5630, 5486] dup_rows_per_dst=[2448, 2724, 2390, 2700, 2820, 2842, 2694, 2454] combine_rows_per_src=[5364, 5518, 5370, 5672, 5700, 5724, 5630, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=160669696 dispatch_meta_bytes=229920 combine_payload_bytes=160669696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160899616 ingress_bytes=185835520 link_bytes=185835520 token_bytes=321569312 weight_movement_bytes=25165824 total_nvlink_bytes=346735136 weight_to_token_ratio=0.078259 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.413993586e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38904 dup_rows=21102 dup_rows_remote=18508 dedup_ratio=0.321991 router_tokens_per_expert=[497, 498, 506, 507, 492, 508, 522, 496, 481, 462, 499, 511, 505, 519, 499, 572, 560, 517, 521, 515, 531, 502, 495, 534, 524, 514, 447, 570, 526, 486, 495, 492, 534, 504, 523, 488, 508, 512, 527, 505, 494, 541, 521, 502, 504, 519, 469, 512, 509, 499, 516, 529, 531, 527, 476, 478, 540, 521, 542, 519, 570, 497, 521, 527] payload_rows_per_dst=[5466, 5526, 5644, 5496, 5582, 5530, 5498, 5692] dup_rows_per_dst=[2586, 2570, 2706, 2612, 2620, 2594, 2632, 2782] combine_rows_per_src=[5466, 5526, 5644, 5496, 5582, 5530, 5498, 5692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=229888 combine_payload_bytes=159866880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160096768 ingress_bytes=185032704 link_bytes=185032704 token_bytes=319963648 weight_movement_bytes=25165824 total_nvlink_bytes=345129472 weight_to_token_ratio=0.078652 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407885084e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159924224 dispatch_meta_bytes=229928 combine_payload_bytes=159924224 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160154152 ingress_bytes=210255872 link_bytes=210255872 token_bytes=320078376 weight_movement_bytes=50331648 total_nvlink_bytes=370410024 weight_to_token_ratio=0.157248 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599804249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38748 dup_rows=21116 dup_rows_remote=18454 dedup_ratio=0.322205 router_tokens_per_expert=[510, 536, 485, 542, 536, 508, 499, 525, 483, 535, 540, 545, 519, 531, 516, 520, 494, 528, 478, 574, 489, 543, 520, 504, 499, 506, 492, 486, 521, 552, 506, 512, 524, 533, 484, 491, 516, 499, 491, 541, 517, 487, 529, 470, 496, 492, 489, 521, 527, 504, 525, 505, 514, 500, 526, 508, 501, 471, 531, 536, 467, 514, 496, 529] payload_rows_per_dst=[5672, 5578, 5596, 5514, 5536, 5468, 5532, 5524] dup_rows_per_dst=[2662, 2800, 2664, 2634, 2622, 2534, 2656, 2544] combine_rows_per_src=[5672, 5578, 5596, 5514, 5536, 5468, 5532, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159907840 dispatch_meta_bytes=229304 combine_payload_bytes=159907840 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160137144 ingress_bytes=235405312 link_bytes=235405312 token_bytes=320044984 weight_movement_bytes=75497472 total_nvlink_bytes=395542456 weight_to_token_ratio=0.235896 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791162429e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158728192 dispatch_meta_bytes=228984 combine_payload_bytes=158728192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158957176 ingress_bytes=183894016 link_bytes=183894016 token_bytes=317685368 weight_movement_bytes=25165824 total_nvlink_bytes=342851192 weight_to_token_ratio=0.079216 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399220984e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159653888 dispatch_meta_bytes=230384 combine_payload_bytes=159653888 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159884272 ingress_bytes=184819712 link_bytes=184819712 token_bytes=319538160 weight_movement_bytes=25165824 total_nvlink_bytes=344703984 weight_to_token_ratio=0.078757 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406264461e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229648 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159580432 ingress_bytes=184516608 link_bytes=184516608 token_bytes=318931216 weight_movement_bytes=25165824 total_nvlink_bytes=344097040 weight_to_token_ratio=0.078907 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403958190e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158711808 dispatch_meta_bytes=228808 combine_payload_bytes=158711808 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=158940616 ingress_bytes=259375104 link_bytes=259375104 token_bytes=317652424 weight_movement_bytes=100663296 total_nvlink_bytes=418315720 weight_to_token_ratio=0.316898 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.973544850e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44552 payload_rows_remote=39028 dup_rows=20984 dup_rows_remote=18392 dedup_ratio=0.320190 router_tokens_per_expert=[479, 506, 507, 535, 486, 500, 505, 509, 556, 524, 504, 504, 466, 554, 544, 523, 489, 530, 509, 493, 539, 492, 514, 476, 492, 526, 524, 502, 515, 495, 534, 470, 523, 500, 500, 503, 511, 504, 505, 526, 497, 556, 532, 505, 518, 492, 553, 502, 486, 488, 518, 551, 497, 517, 518, 511, 530, 508, 543, 481, 536, 511, 541, 503] payload_rows_per_dst=[5446, 5696, 5540, 5524, 5582, 5620, 5520, 5624] dup_rows_per_dst=[2608, 2654, 2544, 2592, 2562, 2690, 2652, 2682] combine_rows_per_src=[5446, 5696, 5540, 5524, 5582, 5620, 5520, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44576 payload_rows_remote=39040 dup_rows=20960 dup_rows_remote=18376 dedup_ratio=0.319824 router_tokens_per_expert=[520, 516, 512, 485, 474, 480, 521, 510, 528, 519, 507, 510, 535, 521, 505, 524, 547, 528, 526, 499, 519, 493, 518, 515, 517, 509, 507, 544, 531, 553, 499, 518, 475, 517, 518, 502, 541, 496, 504, 507, 507, 508, 458, 483, 496, 504, 505, 507, 502, 529, 532, 545, 515, 474, 480, 568, 482, 494, 523, 502, 554, 532, 510, 508] payload_rows_per_dst=[5486, 5630, 5652, 5598, 5536, 5478, 5608, 5588] dup_rows_per_dst=[2550, 2668, 2638, 2758, 2584, 2458, 2682, 2622] combine_rows_per_src=[5486, 5630, 5652, 5598, 5536, 5478, 5608, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44656 payload_rows_remote=39124 dup_rows=20880 dup_rows_remote=18342 dedup_ratio=0.318604 router_tokens_per_expert=[540, 495, 526, 547, 489, 499, 491, 511, 498, 489, 507, 514, 461, 485, 489, 540, 495, 497, 509, 516, 534, 524, 501, 536, 518, 544, 478, 524, 487, 501, 535, 495, 529, 502, 508, 497, 533, 509, 511, 525, 502, 519, 535, 546, 493, 547, 540, 512, 514, 517, 480, 500, 503, 489, 526, 506, 526, 527, 545, 493, 561, 484, 517, 497] payload_rows_per_dst=[5630, 5480, 5576, 5556, 5660, 5640, 5532, 5582] dup_rows_per_dst=[2566, 2486, 2648, 2608, 2568, 2748, 2538, 2718] combine_rows_per_src=[5630, 5480, 5576, 5556, 5660, 5640, 5532, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38912 dup_rows=21076 dup_rows_remote=18408 dedup_ratio=0.321594 router_tokens_per_expert=[503, 469, 519, 513, 517, 479, 561, 512, 523, 520, 500, 502, 489, 485, 535, 554, 471, 542, 485, 530, 478, 498, 527, 526, 492, 519, 522, 509, 509, 521, 502, 548, 517, 496, 517, 519, 501, 527, 498, 531, 544, 505, 512, 518, 522, 490, 464, 525, 455, 526, 509, 535, 526, 528, 480, 543, 546, 501, 503, 517, 546, 518, 498, 491] payload_rows_per_dst=[5552, 5548, 5524, 5548, 5532, 5498, 5612, 5646] dup_rows_per_dst=[2594, 2668, 2590, 2696, 2680, 2662, 2592, 2594] combine_rows_per_src=[5552, 5548, 5524, 5548, 5532, 5498, 5612, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159858688 dispatch_meta_bytes=229680 combine_payload_bytes=159858688 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160088368 ingress_bytes=235356160 link_bytes=235356160 token_bytes=319947056 weight_movement_bytes=75497472 total_nvlink_bytes=395444528 weight_to_token_ratio=0.235969 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790788439e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44684 payload_rows_remote=39046 dup_rows=20852 dup_rows_remote=18108 dedup_ratio=0.318176 router_tokens_per_expert=[538, 564, 511, 458, 486, 532, 487, 532, 544, 549, 539, 448, 511, 523, 510, 472, 520, 512, 521, 504, 548, 520, 532, 534, 498, 512, 540, 451, 526, 527, 530, 503, 513, 544, 499, 524, 516, 521, 503, 504, 493, 500, 545, 470, 505, 471, 502, 530, 470, 480, 513, 533, 513, 486, 533, 486, 508, 500, 517, 518, 533, 523, 519, 514] payload_rows_per_dst=[5632, 5538, 5638, 5582, 5614, 5492, 5590, 5598] dup_rows_per_dst=[2584, 2654, 2744, 2592, 2634, 2540, 2438, 2666] combine_rows_per_src=[5632, 5538, 5638, 5582, 5614, 5492, 5590, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38780 dup_rows=21166 dup_rows_remote=18534 dedup_ratio=0.322968 router_tokens_per_expert=[492, 531, 514, 474, 491, 501, 503, 488, 519, 498, 531, 553, 519, 525, 524, 536, 507, 500, 459, 485, 542, 497, 503, 505, 528, 516, 521, 523, 506, 472, 524, 529, 526, 509, 523, 483, 497, 486, 522, 481, 539, 548, 525, 509, 499, 491, 580, 508, 546, 500, 528, 534, 526, 540, 500, 533, 511, 507, 473, 511, 508, 491, 494, 524] payload_rows_per_dst=[5510, 5414, 5452, 5596, 5646, 5568, 5594, 5590] dup_rows_per_dst=[2478, 2494, 2504, 2654, 2726, 2830, 2848, 2632] combine_rows_per_src=[5510, 5414, 5452, 5596, 5646, 5568, 5594, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44514 payload_rows_remote=39124 dup_rows=21022 dup_rows_remote=18626 dedup_ratio=0.320770 router_tokens_per_expert=[485, 513, 532, 500, 531, 531, 572, 525, 486, 543, 523, 531, 534, 535, 489, 532, 491, 551, 515, 539, 538, 540, 470, 496, 519, 507, 510, 570, 532, 509, 509, 557, 529, 507, 513, 492, 537, 522, 456, 486, 452, 485, 481, 491, 487, 465, 525, 507, 471, 506, 487, 521, 531, 526, 534, 532, 514, 520, 497, 496, 502, 509, 489, 483] payload_rows_per_dst=[5646, 5648, 5600, 5652, 5510, 5390, 5558, 5510] dup_rows_per_dst=[2732, 2698, 2680, 2774, 2574, 2396, 2658, 2510] combine_rows_per_src=[5646, 5648, 5600, 5652, 5510, 5390, 5558, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160251904 dispatch_meta_bytes=229864 combine_payload_bytes=160251904 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160481768 ingress_bytes=210583552 link_bytes=210583552 token_bytes=320733672 weight_movement_bytes=50331648 total_nvlink_bytes=371065320 weight_to_token_ratio=0.156927 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602297515e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159907840 dispatch_meta_bytes=229664 combine_payload_bytes=159907840 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160137504 ingress_bytes=185073664 link_bytes=185073664 token_bytes=320045344 weight_movement_bytes=25165824 total_nvlink_bytes=345211168 weight_to_token_ratio=0.078632 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408196742e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44678 payload_rows_remote=39014 dup_rows=20858 dup_rows_remote=18000 dedup_ratio=0.318268 router_tokens_per_expert=[514, 522, 561, 498, 515, 487, 528, 527, 478, 536, 499, 517, 478, 527, 530, 534, 470, 524, 492, 536, 536, 530, 508, 507, 515, 511, 501, 510, 552, 496, 494, 464, 524, 514, 508, 526, 520, 491, 491, 490, 502, 551, 541, 539, 520, 515, 526, 510, 509, 525, 521, 502, 529, 534, 493, 503, 518, 461, 473, 532, 485, 487, 525, 506] payload_rows_per_dst=[5664, 5586, 5606, 5392, 5546, 5756, 5634, 5494] dup_rows_per_dst=[2858, 2674, 2600, 2414, 2582, 2770, 2480, 2480] combine_rows_per_src=[5664, 5586, 5606, 5392, 5546, 5756, 5634, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159383552 dispatch_meta_bytes=229280 combine_payload_bytes=159383552 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159612832 ingress_bytes=234881024 link_bytes=234881024 token_bytes=318996384 weight_movement_bytes=75497472 total_nvlink_bytes=394493856 weight_to_token_ratio=0.236672 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787173203e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159932416 dispatch_meta_bytes=228616 combine_payload_bytes=159932416 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160161032 ingress_bytes=210264064 link_bytes=210264064 token_bytes=320093448 weight_movement_bytes=50331648 total_nvlink_bytes=370425096 weight_to_token_ratio=0.157240 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599866580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=229256 combine_payload_bytes=158842880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159072136 ingress_bytes=184008704 link_bytes=184008704 token_bytes=317915016 weight_movement_bytes=25165824 total_nvlink_bytes=343080840 weight_to_token_ratio=0.079159 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400093627e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160251904 dispatch_meta_bytes=231000 combine_payload_bytes=160251904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160482904 ingress_bytes=185417728 link_bytes=185417728 token_bytes=320734808 weight_movement_bytes=25165824 total_nvlink_bytes=345900632 weight_to_token_ratio=0.078463 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410814672e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159801344 dispatch_meta_bytes=228056 combine_payload_bytes=159801344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160029400 ingress_bytes=184967168 link_bytes=184967168 token_bytes=319830744 weight_movement_bytes=25165824 total_nvlink_bytes=344996568 weight_to_token_ratio=0.078685 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407386431e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44656 payload_rows_remote=39190 dup_rows=20880 dup_rows_remote=18260 dedup_ratio=0.318604 router_tokens_per_expert=[516, 513, 483, 518, 525, 536, 465, 527, 497, 477, 531, 537, 503, 539, 530, 492, 511, 488, 521, 520, 520, 515, 545, 543, 498, 513, 498, 500, 497, 514, 495, 528, 505, 525, 526, 501, 496, 535, 513, 531, 509, 545, 503, 531, 502, 521, 494, 497, 535, 503, 536, 490, 511, 500, 519, 495, 508, 506, 512, 492, 507, 528, 471, 526] payload_rows_per_dst=[5540, 5578, 5612, 5466, 5668, 5600, 5636, 5556] dup_rows_per_dst=[2626, 2634, 2714, 2620, 2596, 2604, 2542, 2544] combine_rows_per_src=[5540, 5578, 5612, 5466, 5668, 5600, 5636, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44514 payload_rows_remote=38958 dup_rows=21022 dup_rows_remote=18344 dedup_ratio=0.320770 router_tokens_per_expert=[524, 500, 491, 480, 498, 508, 469, 510, 525, 541, 505, 515, 532, 525, 509, 480, 467, 498, 542, 537, 534, 496, 488, 531, 511, 514, 492, 560, 492, 521, 507, 528, 546, 489, 480, 530, 515, 556, 512, 516, 494, 522, 497, 528, 526, 505, 483, 502, 529, 495, 522, 525, 497, 502, 519, 528, 500, 554, 512, 532, 487, 509, 526, 500] payload_rows_per_dst=[5434, 5588, 5584, 5560, 5534, 5532, 5556, 5726] dup_rows_per_dst=[2526, 2676, 2602, 2690, 2754, 2582, 2678, 2514] combine_rows_per_src=[5434, 5588, 5584, 5560, 5534, 5532, 5556, 5726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=39074 dup_rows=20976 dup_rows_remote=18336 dedup_ratio=0.320068 router_tokens_per_expert=[493, 542, 491, 522, 451, 514, 526, 493, 482, 535, 516, 535, 531, 498, 506, 514, 511, 497, 547, 528, 493, 532, 502, 542, 458, 507, 492, 476, 527, 529, 495, 512, 518, 516, 527, 478, 520, 528, 488, 488, 488, 494, 522, 555, 555, 523, 483, 514, 492, 503, 552, 496, 499, 537, 494, 527, 549, 532, 501, 531, 539, 518, 510, 494] payload_rows_per_dst=[5500, 5566, 5622, 5434, 5486, 5658, 5610, 5684] dup_rows_per_dst=[2564, 2668, 2682, 2558, 2640, 2610, 2590, 2664] combine_rows_per_src=[5500, 5566, 5622, 5434, 5486, 5658, 5610, 5684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160522240 dispatch_meta_bytes=229800 combine_payload_bytes=160522240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160752040 ingress_bytes=185688064 link_bytes=185688064 token_bytes=321274280 weight_movement_bytes=25165824 total_nvlink_bytes=346440104 weight_to_token_ratio=0.078331 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412871616e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44780 payload_rows_remote=39168 dup_rows=20756 dup_rows_remote=18142 dedup_ratio=0.316711 router_tokens_per_expert=[501, 510, 501, 526, 498, 498, 553, 511, 504, 533, 523, 494, 565, 490, 504, 500, 519, 523, 526, 532, 458, 499, 516, 481, 496, 523, 523, 554, 500, 532, 482, 503, 522, 491, 500, 494, 514, 525, 521, 527, 492, 552, 509, 499, 527, 503, 509, 524, 542, 502, 527, 495, 480, 545, 496, 500, 524, 510, 560, 482, 516, 481, 500, 521] payload_rows_per_dst=[5610, 5612, 5554, 5606, 5544, 5646, 5614, 5594] dup_rows_per_dst=[2586, 2614, 2554, 2620, 2644, 2584, 2560, 2594] combine_rows_per_src=[5610, 5612, 5554, 5606, 5544, 5646, 5614, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44292 payload_rows_remote=38796 dup_rows=21244 dup_rows_remote=18590 dedup_ratio=0.324158 router_tokens_per_expert=[495, 479, 502, 535, 487, 513, 482, 488, 535, 503, 523, 513, 501, 515, 513, 494, 475, 498, 512, 518, 517, 519, 504, 532, 496, 568, 501, 541, 513, 484, 517, 504, 516, 516, 516, 537, 562, 536, 484, 519, 517, 486, 515, 503, 486, 490, 552, 502, 526, 527, 530, 474, 524, 529, 509, 525, 567, 516, 481, 526, 526, 490, 502, 502] payload_rows_per_dst=[5394, 5506, 5496, 5594, 5626, 5524, 5582, 5570] dup_rows_per_dst=[2568, 2688, 2654, 2654, 2746, 2578, 2706, 2650] combine_rows_per_src=[5394, 5506, 5496, 5594, 5626, 5524, 5582, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159571968 dispatch_meta_bytes=229208 combine_payload_bytes=159571968 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159801176 ingress_bytes=235069440 link_bytes=235069440 token_bytes=319373144 weight_movement_bytes=75497472 total_nvlink_bytes=394870616 weight_to_token_ratio=0.236393 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788606831e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44708 payload_rows_remote=39050 dup_rows=20828 dup_rows_remote=18080 dedup_ratio=0.317810 router_tokens_per_expert=[494, 503, 486, 521, 497, 502, 519, 515, 473, 512, 503, 519, 501, 539, 517, 480, 529, 501, 480, 549, 514, 461, 507, 522, 520, 513, 538, 561, 493, 542, 528, 512, 512, 476, 494, 528, 508, 539, 495, 480, 503, 524, 494, 514, 499, 518, 548, 478, 541, 510, 509, 525, 497, 535, 528, 536, 521, 534, 519, 505, 532, 480, 542, 493] payload_rows_per_dst=[5600, 5574, 5548, 5712, 5390, 5556, 5670, 5658] dup_rows_per_dst=[2474, 2506, 2578, 2702, 2414, 2692, 2714, 2748] combine_rows_per_src=[5600, 5574, 5548, 5712, 5390, 5556, 5670, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44526 payload_rows_remote=38964 dup_rows=21010 dup_rows_remote=18382 dedup_ratio=0.320587 router_tokens_per_expert=[491, 546, 533, 512, 480, 531, 524, 534, 485, 484, 517, 526, 478, 490, 534, 504, 496, 501, 551, 532, 523, 517, 498, 514, 491, 536, 477, 509, 521, 489, 546, 494, 496, 503, 513, 509, 527, 484, 529, 535, 506, 536, 482, 493, 507, 500, 529, 542, 519, 513, 489, 519, 495, 516, 513, 478, 527, 530, 522, 521, 509, 511, 543, 508] payload_rows_per_dst=[5636, 5510, 5552, 5534, 5594, 5562, 5556, 5582] dup_rows_per_dst=[2666, 2526, 2712, 2592, 2598, 2628, 2528, 2760] combine_rows_per_src=[5636, 5510, 5552, 5534, 5594, 5562, 5556, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160047104 dispatch_meta_bytes=229640 combine_payload_bytes=160047104 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=160276744 ingress_bytes=260710400 link_bytes=260710400 token_bytes=320323848 weight_movement_bytes=100663296 total_nvlink_bytes=420987144 weight_to_token_ratio=0.314255 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.983704910e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38640 dup_rows=21108 dup_rows_remote=18084 dedup_ratio=0.322083 router_tokens_per_expert=[468, 506, 510, 541, 525, 497, 484, 538, 485, 508, 555, 519, 499, 542, 502, 535, 512, 508, 489, 506, 464, 507, 485, 523, 480, 475, 497, 542, 484, 482, 530, 490, 552, 516, 533, 529, 477, 522, 539, 483, 513, 525, 497, 504, 493, 533, 531, 494, 538, 505, 542, 525, 487, 499, 545, 515, 533, 511, 544, 552, 517, 519, 495, 512] payload_rows_per_dst=[5788, 5590, 5488, 5612, 5568, 5532, 5464, 5386] dup_rows_per_dst=[3024, 2700, 2584, 2582, 2734, 2648, 2530, 2306] combine_rows_per_src=[5788, 5590, 5488, 5612, 5568, 5532, 5464, 5386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160432128 dispatch_meta_bytes=229240 combine_payload_bytes=160432128 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160661368 ingress_bytes=185597952 link_bytes=185597952 token_bytes=321093496 weight_movement_bytes=25165824 total_nvlink_bytes=346259320 weight_to_token_ratio=0.078375 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412185968e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158908416 dispatch_meta_bytes=229544 combine_payload_bytes=158908416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159137960 ingress_bytes=184074240 link_bytes=184074240 token_bytes=318046376 weight_movement_bytes=25165824 total_nvlink_bytes=343212200 weight_to_token_ratio=0.079126 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400592280e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159948800 dispatch_meta_bytes=228520 combine_payload_bytes=159948800 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160177320 ingress_bytes=210280448 link_bytes=210280448 token_bytes=320126120 weight_movement_bytes=50331648 total_nvlink_bytes=370457768 weight_to_token_ratio=0.157224 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599991244e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229384 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159825928 ingress_bytes=184762368 link_bytes=184762368 token_bytes=319422472 weight_movement_bytes=25165824 total_nvlink_bytes=344588296 weight_to_token_ratio=0.078785 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405828139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38964 dup_rows=21124 dup_rows_remote=18478 dedup_ratio=0.322327 router_tokens_per_expert=[514, 516, 552, 506, 511, 508, 532, 485, 517, 514, 514, 512, 507, 507, 519, 536, 537, 537, 524, 506, 500, 502, 516, 485, 503, 513, 513, 512, 496, 482, 507, 521, 484, 539, 474, 522, 506, 496, 519, 500, 508, 517, 545, 486, 514, 536, 519, 517, 507, 517, 484, 530, 470, 506, 488, 549, 521, 504, 547, 485, 528, 547, 496, 503] payload_rows_per_dst=[5560, 5566, 5552, 5448, 5454, 5690, 5512, 5630] dup_rows_per_dst=[2688, 2686, 2662, 2646, 2626, 2594, 2590, 2632] combine_rows_per_src=[5560, 5566, 5552, 5448, 5454, 5690, 5512, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158269440 dispatch_meta_bytes=226896 combine_payload_bytes=158269440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158496336 ingress_bytes=183435264 link_bytes=183435264 token_bytes=316765776 weight_movement_bytes=25165824 total_nvlink_bytes=341931600 weight_to_token_ratio=0.079446 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.395730411e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38904 dup_rows=21046 dup_rows_remote=18420 dedup_ratio=0.321136 router_tokens_per_expert=[509, 489, 537, 532, 524, 526, 514, 520, 509, 497, 531, 501, 530, 568, 516, 497, 533, 519, 520, 504, 481, 495, 512, 507, 497, 493, 510, 522, 530, 503, 475, 534, 499, 483, 535, 493, 522, 478, 543, 523, 509, 506, 487, 507, 528, 511, 516, 516, 499, 525, 500, 492, 507, 538, 526, 519, 549, 513, 506, 482, 513, 504, 498, 506] payload_rows_per_dst=[5620, 5550, 5538, 5568, 5536, 5514, 5586, 5578] dup_rows_per_dst=[2682, 2748, 2604, 2560, 2616, 2646, 2626, 2564] combine_rows_per_src=[5620, 5550, 5538, 5568, 5536, 5514, 5586, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44196 payload_rows_remote=38610 dup_rows=21340 dup_rows_remote=18656 dedup_ratio=0.325623 router_tokens_per_expert=[544, 500, 500, 490, 494, 515, 516, 481, 497, 515, 524, 512, 551, 506, 516, 503, 540, 545, 483, 492, 508, 472, 552, 499, 503, 532, 522, 521, 508, 537, 488, 544, 529, 556, 528, 482, 517, 474, 519, 530, 513, 487, 488, 539, 484, 495, 512, 537, 499, 483, 508, 497, 533, 523, 495, 525, 480, 494, 510, 539, 515, 527, 533, 507] payload_rows_per_dst=[5446, 5522, 5524, 5596, 5586, 5486, 5512, 5524] dup_rows_per_dst=[2634, 2726, 2658, 2714, 2684, 2624, 2614, 2686] combine_rows_per_src=[5446, 5522, 5524, 5596, 5586, 5486, 5512, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229768 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159826312 ingress_bytes=184762368 link_bytes=184762368 token_bytes=319422856 weight_movement_bytes=25165824 total_nvlink_bytes=344588680 weight_to_token_ratio=0.078785 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405828139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38982 dup_rows=21068 dup_rows_remote=18476 dedup_ratio=0.321472 router_tokens_per_expert=[518, 539, 551, 488, 544, 506, 492, 513, 523, 541, 490, 489, 508, 512, 502, 474, 538, 547, 498, 516, 519, 517, 553, 489, 525, 518, 495, 514, 538, 508, 490, 509, 546, 526, 498, 482, 535, 554, 513, 534, 500, 492, 505, 500, 501, 485, 482, 521, 494, 525, 486, 503, 532, 492, 529, 519, 481, 522, 531, 507, 494, 524, 471, 520] payload_rows_per_dst=[5592, 5486, 5642, 5528, 5698, 5472, 5542, 5508] dup_rows_per_dst=[2710, 2592, 2712, 2666, 2678, 2500, 2618, 2592] combine_rows_per_src=[5592, 5486, 5642, 5528, 5698, 5472, 5542, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229296 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159580080 ingress_bytes=184516608 link_bytes=184516608 token_bytes=318930864 weight_movement_bytes=25165824 total_nvlink_bytes=344096688 weight_to_token_ratio=0.078907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403958190e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44364 payload_rows_remote=38840 dup_rows=21172 dup_rows_remote=18580 dedup_ratio=0.323059 router_tokens_per_expert=[519, 470, 542, 537, 531, 500, 485, 480, 487, 510, 500, 505, 511, 484, 495, 509, 497, 483, 517, 505, 523, 502, 518, 513, 517, 561, 504, 519, 480, 502, 540, 512, 494, 534, 490, 501, 530, 550, 527, 514, 522, 544, 497, 512, 446, 526, 521, 539, 517, 505, 523, 504, 505, 544, 505, 533, 532, 499, 544, 478, 503, 485, 545, 541] payload_rows_per_dst=[5552, 5488, 5524, 5532, 5560, 5552, 5568, 5588] dup_rows_per_dst=[2576, 2514, 2592, 2738, 2720, 2662, 2704, 2666] combine_rows_per_src=[5552, 5488, 5524, 5532, 5560, 5552, 5568, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38780 dup_rows=21098 dup_rows_remote=18532 dedup_ratio=0.321930 router_tokens_per_expert=[553, 517, 534, 528, 512, 530, 507, 490, 502, 555, 515, 506, 533, 527, 535, 476, 481, 510, 560, 552, 516, 521, 534, 530, 530, 502, 485, 492, 536, 498, 488, 547, 503, 533, 496, 466, 503, 514, 518, 495, 524, 491, 504, 492, 524, 525, 507, 479, 493, 505, 521, 499, 517, 477, 492, 493, 551, 487, 473, 528, 537, 506, 523, 490] payload_rows_per_dst=[5404, 5578, 5512, 5558, 5506, 5686, 5536, 5658] dup_rows_per_dst=[2330, 2674, 2584, 2644, 2596, 2934, 2770, 2566] combine_rows_per_src=[5404, 5578, 5512, 5558, 5506, 5686, 5536, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44550 payload_rows_remote=39058 dup_rows=20986 dup_rows_remote=18452 dedup_ratio=0.320221 router_tokens_per_expert=[517, 527, 512, 550, 510, 521, 505, 494, 543, 492, 490, 503, 475, 529, 500, 505, 481, 525, 564, 508, 517, 489, 530, 475, 490, 496, 517, 546, 505, 539, 545, 527, 488, 510, 541, 529, 516, 498, 516, 503, 504, 502, 511, 481, 489, 508, 494, 524, 480, 502, 532, 513, 546, 535, 485, 529, 489, 496, 491, 542, 541, 533, 490, 523] payload_rows_per_dst=[5610, 5498, 5562, 5610, 5594, 5492, 5634, 5550] dup_rows_per_dst=[2662, 2576, 2616, 2720, 2608, 2534, 2610, 2660] combine_rows_per_src=[5610, 5498, 5562, 5610, 5594, 5492, 5634, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158146560 dispatch_meta_bytes=229064 combine_payload_bytes=158146560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158375624 ingress_bytes=183312384 link_bytes=183312384 token_bytes=316522184 weight_movement_bytes=25165824 total_nvlink_bytes=341688008 weight_to_token_ratio=0.079507 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.394795436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44642 payload_rows_remote=38978 dup_rows=20894 dup_rows_remote=18088 dedup_ratio=0.318817 router_tokens_per_expert=[571, 522, 540, 531, 533, 492, 495, 546, 537, 538, 536, 529, 503, 509, 518, 501, 537, 516, 515, 497, 491, 514, 491, 513, 496, 519, 488, 482, 518, 500, 501, 500, 486, 504, 512, 531, 499, 475, 502, 528, 514, 502, 514, 514, 497, 534, 513, 487, 513, 511, 501, 472, 540, 522, 511, 487, 518, 508, 514, 502, 517, 534, 487, 540] payload_rows_per_dst=[5664, 5668, 5632, 5546, 5462, 5562, 5584, 5524] dup_rows_per_dst=[2806, 2664, 2622, 2462, 2612, 2588, 2530, 2610] combine_rows_per_src=[5664, 5668, 5632, 5546, 5462, 5562, 5584, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229832 combine_payload_bytes=159670272 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159900104 ingress_bytes=210001920 link_bytes=210001920 token_bytes=319570376 weight_movement_bytes=50331648 total_nvlink_bytes=369902024 weight_to_token_ratio=0.157498 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597871967e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=229680 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159318320 ingress_bytes=209420288 link_bytes=209420288 token_bytes=318406960 weight_movement_bytes=50331648 total_nvlink_bytes=368738608 weight_to_token_ratio=0.158073 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593446420e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=229248 combine_payload_bytes=158842880 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159072128 ingress_bytes=209174528 link_bytes=209174528 token_bytes=317915008 weight_movement_bytes=50331648 total_nvlink_bytes=368246656 weight_to_token_ratio=0.158318 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591576470e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38938 dup_rows=21052 dup_rows_remote=18488 dedup_ratio=0.321228 router_tokens_per_expert=[499, 541, 505, 536, 522, 505, 537, 517, 530, 508, 559, 487, 503, 505, 499, 511, 491, 485, 499, 535, 537, 520, 483, 553, 508, 470, 529, 528, 537, 482, 487, 514, 494, 496, 486, 534, 504, 504, 502, 512, 497, 540, 533, 456, 547, 496, 527, 550, 520, 487, 508, 550, 512, 523, 553, 508, 496, 479, 509, 516, 485, 499, 488, 535] payload_rows_per_dst=[5620, 5512, 5522, 5546, 5562, 5646, 5604, 5472] dup_rows_per_dst=[2704, 2692, 2684, 2564, 2502, 2646, 2718, 2542] combine_rows_per_src=[5620, 5512, 5522, 5546, 5562, 5646, 5604, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159981568 dispatch_meta_bytes=230040 combine_payload_bytes=159981568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160211608 ingress_bytes=185147392 link_bytes=185147392 token_bytes=320193176 weight_movement_bytes=25165824 total_nvlink_bytes=345359000 weight_to_token_ratio=0.078596 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408757727e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159653888 dispatch_meta_bytes=228264 combine_payload_bytes=159653888 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159882152 ingress_bytes=260317184 link_bytes=260317184 token_bytes=319536040 weight_movement_bytes=100663296 total_nvlink_bytes=420199336 weight_to_token_ratio=0.315030 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.980712990e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38970 dup_rows=21026 dup_rows_remote=18324 dedup_ratio=0.320831 router_tokens_per_expert=[560, 481, 502, 487, 542, 512, 519, 515, 545, 543, 526, 509, 492, 486, 539, 514, 497, 498, 537, 473, 481, 504, 507, 479, 472, 493, 506, 555, 527, 526, 503, 504, 506, 523, 527, 553, 483, 518, 567, 488, 523, 523, 507, 522, 502, 505, 494, 518, 512, 530, 483, 546, 518, 501, 517, 514, 543, 509, 499, 489, 497, 515, 499, 503] payload_rows_per_dst=[5562, 5610, 5462, 5548, 5658, 5566, 5540, 5564] dup_rows_per_dst=[2674, 2698, 2490, 2624, 2672, 2622, 2702, 2544] combine_rows_per_src=[5562, 5610, 5462, 5548, 5658, 5566, 5540, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159490048 dispatch_meta_bytes=229704 combine_payload_bytes=159490048 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159719752 ingress_bytes=184655872 link_bytes=184655872 token_bytes=319209800 weight_movement_bytes=25165824 total_nvlink_bytes=344375624 weight_to_token_ratio=0.078838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405017828e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44406 payload_rows_remote=38838 dup_rows=21130 dup_rows_remote=18568 dedup_ratio=0.322418 router_tokens_per_expert=[520, 521, 539, 522, 545, 519, 502, 498, 492, 523, 484, 519, 506, 505, 505, 547, 541, 508, 496, 535, 543, 511, 505, 554, 518, 529, 514, 511, 460, 512, 513, 511, 513, 480, 495, 467, 554, 515, 494, 547, 515, 526, 521, 511, 520, 476, 492, 515, 539, 487, 490, 525, 487, 538, 580, 455, 496, 483, 515, 531, 511, 495, 477, 510] payload_rows_per_dst=[5608, 5526, 5628, 5492, 5568, 5652, 5494, 5438] dup_rows_per_dst=[2724, 2636, 2758, 2644, 2562, 2500, 2708, 2598] combine_rows_per_src=[5608, 5526, 5628, 5492, 5568, 5652, 5494, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44740 payload_rows_remote=39280 dup_rows=20796 dup_rows_remote=18306 dedup_ratio=0.317322 router_tokens_per_expert=[554, 527, 484, 524, 575, 522, 528, 513, 501, 511, 496, 477, 515, 508, 465, 502, 488, 488, 474, 474, 529, 523, 481, 531, 547, 512, 505, 517, 518, 491, 525, 526, 542, 553, 466, 519, 520, 502, 548, 469, 504, 524, 479, 479, 497, 514, 528, 498, 526, 508, 509, 541, 521, 535, 509, 530, 518, 524, 501, 501, 529, 502, 521, 520] payload_rows_per_dst=[5762, 5460, 5518, 5602, 5646, 5512, 5654, 5586] dup_rows_per_dst=[2692, 2490, 2458, 2680, 2592, 2534, 2704, 2646] combine_rows_per_src=[5762, 5460, 5518, 5602, 5646, 5512, 5654, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159621120 dispatch_meta_bytes=229176 combine_payload_bytes=159621120 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159850296 ingress_bytes=235118592 link_bytes=235118592 token_bytes=319471416 weight_movement_bytes=75497472 total_nvlink_bytes=394968888 weight_to_token_ratio=0.236320 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788980821e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44612 payload_rows_remote=39002 dup_rows=20924 dup_rows_remote=18292 dedup_ratio=0.319275 router_tokens_per_expert=[514, 509, 545, 509, 530, 506, 541, 522, 527, 504, 525, 486, 509, 472, 516, 539, 494, 541, 517, 502, 523, 523, 520, 501, 500, 498, 486, 501, 524, 491, 506, 491, 526, 539, 527, 472, 525, 525, 496, 501, 498, 498, 518, 505, 559, 525, 501, 504, 515, 530, 534, 509, 546, 496, 516, 510, 504, 503, 513, 476, 485, 522, 507, 511] payload_rows_per_dst=[5572, 5604, 5610, 5460, 5648, 5624, 5610, 5484] dup_rows_per_dst=[2780, 2552, 2632, 2534, 2574, 2592, 2702, 2558] combine_rows_per_src=[5572, 5604, 5610, 5460, 5648, 5624, 5610, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44292 payload_rows_remote=38508 dup_rows=21244 dup_rows_remote=18154 dedup_ratio=0.324158 router_tokens_per_expert=[544, 552, 503, 507, 514, 505, 491, 505, 539, 534, 503, 527, 513, 501, 504, 488, 569, 502, 529, 501, 488, 521, 493, 511, 513, 497, 509, 516, 534, 501, 521, 484, 505, 505, 524, 513, 491, 501, 481, 500, 531, 536, 523, 503, 524, 473, 508, 502, 524, 531, 509, 529, 482, 503, 471, 551, 492, 506, 470, 547, 541, 520, 515, 538] payload_rows_per_dst=[5306, 5614, 5570, 5474, 5512, 5736, 5296, 5784] dup_rows_per_dst=[2370, 2694, 2658, 2676, 2584, 2876, 2296, 3090] combine_rows_per_src=[5306, 5614, 5570, 5474, 5512, 5736, 5296, 5784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159080448 dispatch_meta_bytes=229624 combine_payload_bytes=159080448 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159310072 ingress_bytes=209412096 link_bytes=209412096 token_bytes=318390520 weight_movement_bytes=50331648 total_nvlink_bytes=368722168 weight_to_token_ratio=0.158081 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593384088e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44682 payload_rows_remote=39240 dup_rows=20854 dup_rows_remote=18402 dedup_ratio=0.318207 router_tokens_per_expert=[549, 553, 471, 536, 485, 506, 490, 504, 472, 482, 543, 505, 562, 492, 514, 531, 501, 520, 539, 494, 516, 484, 506, 537, 554, 504, 529, 521, 481, 524, 518, 501, 502, 524, 547, 521, 535, 546, 500, 567, 467, 515, 500, 498, 489, 483, 520, 475, 507, 544, 485, 482, 540, 531, 552, 542, 505, 493, 508, 492, 485, 496, 515, 478] payload_rows_per_dst=[5610, 5616, 5596, 5658, 5674, 5442, 5632, 5454] dup_rows_per_dst=[2578, 2586, 2598, 2606, 2810, 2452, 2734, 2490] combine_rows_per_src=[5610, 5616, 5596, 5658, 5674, 5442, 5632, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44316 payload_rows_remote=38878 dup_rows=21220 dup_rows_remote=18684 dedup_ratio=0.323792 router_tokens_per_expert=[499, 497, 492, 521, 488, 507, 491, 492, 539, 515, 533, 521, 503, 516, 514, 482, 479, 508, 546, 497, 512, 517, 510, 506, 532, 498, 500, 534, 517, 519, 499, 489, 533, 526, 524, 554, 486, 494, 503, 492, 510, 507, 556, 472, 514, 535, 476, 498, 527, 529, 523, 520, 539, 533, 512, 489, 536, 532, 531, 503, 520, 487, 531, 503] payload_rows_per_dst=[5438, 5596, 5574, 5602, 5454, 5562, 5554, 5536] dup_rows_per_dst=[2536, 2650, 2778, 2826, 2518, 2574, 2588, 2750] combine_rows_per_src=[5438, 5596, 5574, 5602, 5454, 5562, 5554, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160890880 dispatch_meta_bytes=230344 combine_payload_bytes=160890880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=161121224 ingress_bytes=186056704 link_bytes=186056704 token_bytes=322012104 weight_movement_bytes=25165824 total_nvlink_bytes=347177928 weight_to_token_ratio=0.078152 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.415676541e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44612 payload_rows_remote=38964 dup_rows=20924 dup_rows_remote=18306 dedup_ratio=0.319275 router_tokens_per_expert=[540, 520, 506, 538, 494, 500, 533, 500, 499, 534, 535, 511, 481, 501, 483, 506, 500, 529, 474, 490, 496, 493, 538, 526, 510, 513, 534, 497, 522, 533, 499, 525, 515, 536, 511, 521, 494, 512, 515, 509, 520, 501, 507, 536, 479, 503, 543, 493, 517, 525, 513, 501, 516, 474, 509, 532, 489, 529, 515, 521, 527, 519, 503, 523] payload_rows_per_dst=[5626, 5526, 5526, 5648, 5642, 5580, 5502, 5562] dup_rows_per_dst=[2636, 2574, 2566, 2618, 2584, 2584, 2672, 2690] combine_rows_per_src=[5626, 5526, 5526, 5648, 5642, 5580, 5502, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159752192 dispatch_meta_bytes=229176 combine_payload_bytes=159752192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159981368 ingress_bytes=184918016 link_bytes=184918016 token_bytes=319733560 weight_movement_bytes=25165824 total_nvlink_bytes=344899384 weight_to_token_ratio=0.078709 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407012441e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=157728768 dispatch_meta_bytes=226648 combine_payload_bytes=157728768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=157955416 ingress_bytes=182894592 link_bytes=182894592 token_bytes=315684184 weight_movement_bytes=25165824 total_nvlink_bytes=340850008 weight_to_token_ratio=0.079718 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.391616522e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160727040 dispatch_meta_bytes=230568 combine_payload_bytes=160727040 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160957608 ingress_bytes=185892864 link_bytes=185892864 token_bytes=321684648 weight_movement_bytes=25165824 total_nvlink_bytes=346850472 weight_to_token_ratio=0.078231 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.414429908e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=230248 combine_payload_bytes=159244288 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159474536 ingress_bytes=259907584 link_bytes=259907584 token_bytes=318718824 weight_movement_bytes=100663296 total_nvlink_bytes=419382120 weight_to_token_ratio=0.315837 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.977596408e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44374 payload_rows_remote=38802 dup_rows=21162 dup_rows_remote=18484 dedup_ratio=0.322906 router_tokens_per_expert=[475, 504, 512, 515, 514, 511, 538, 506, 512, 499, 529, 524, 484, 484, 496, 551, 499, 512, 559, 504, 515, 523, 579, 507, 501, 553, 512, 479, 534, 502, 514, 519, 525, 463, 535, 493, 520, 488, 518, 478, 493, 459, 492, 524, 516, 478, 494, 519, 501, 518, 525, 532, 526, 484, 514, 525, 530, 522, 521, 513, 534, 525, 509, 528] payload_rows_per_dst=[5560, 5558, 5568, 5508, 5484, 5406, 5572, 5718] dup_rows_per_dst=[2590, 2600, 2828, 2720, 2556, 2544, 2678, 2646] combine_rows_per_src=[5560, 5558, 5568, 5508, 5484, 5406, 5572, 5718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44252 payload_rows_remote=38760 dup_rows=21284 dup_rows_remote=18512 dedup_ratio=0.324768 router_tokens_per_expert=[527, 516, 502, 505, 501, 476, 506, 553, 507, 488, 531, 531, 505, 525, 501, 498, 524, 514, 481, 556, 521, 514, 517, 493, 555, 513, 516, 499, 496, 489, 528, 503, 537, 518, 524, 519, 482, 523, 503, 526, 531, 486, 482, 507, 514, 514, 507, 512, 537, 503, 526, 496, 487, 511, 513, 493, 542, 531, 528, 479, 517, 515, 515, 499] payload_rows_per_dst=[5508, 5582, 5514, 5566, 5492, 5494, 5468, 5628] dup_rows_per_dst=[2664, 2590, 2726, 2632, 2772, 2612, 2664, 2624] combine_rows_per_src=[5508, 5582, 5514, 5566, 5492, 5494, 5468, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229080 combine_payload_bytes=159596544 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159825624 ingress_bytes=209928192 link_bytes=209928192 token_bytes=319422168 weight_movement_bytes=50331648 total_nvlink_bytes=369753816 weight_to_token_ratio=0.157571 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597310982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44608 payload_rows_remote=39072 dup_rows=20928 dup_rows_remote=18402 dedup_ratio=0.319336 router_tokens_per_expert=[540, 511, 510, 507, 517, 541, 501, 525, 521, 540, 525, 503, 486, 454, 520, 482, 536, 538, 483, 522, 483, 537, 551, 500, 499, 501, 502, 509, 500, 532, 509, 502, 514, 474, 474, 524, 548, 502, 542, 529, 516, 487, 474, 533, 521, 520, 526, 485, 506, 484, 551, 504, 523, 518, 502, 545, 491, 512, 503, 505, 535, 514, 510, 509] payload_rows_per_dst=[5626, 5536, 5590, 5550, 5592, 5574, 5596, 5544] dup_rows_per_dst=[2678, 2526, 2710, 2558, 2622, 2550, 2670, 2614] combine_rows_per_src=[5626, 5536, 5590, 5550, 5592, 5574, 5596, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158932992 dispatch_meta_bytes=229144 combine_payload_bytes=158932992 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159162136 ingress_bytes=234430464 link_bytes=234430464 token_bytes=318095128 weight_movement_bytes=75497472 total_nvlink_bytes=393592600 weight_to_token_ratio=0.237342 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783744962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=229088 combine_payload_bytes=158760960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158990048 ingress_bytes=183926784 link_bytes=183926784 token_bytes=317751008 weight_movement_bytes=25165824 total_nvlink_bytes=342916832 weight_to_token_ratio=0.079200 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399470311e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=38988 dup_rows=20976 dup_rows_remote=18362 dedup_ratio=0.320068 router_tokens_per_expert=[471, 543, 498, 508, 508, 522, 535, 492, 498, 520, 502, 480, 502, 542, 517, 517, 523, 507, 553, 511, 531, 485, 496, 487, 508, 490, 525, 532, 512, 536, 490, 525, 493, 520, 521, 512, 523, 514, 503, 494, 484, 545, 513, 494, 496, 505, 516, 506, 503, 525, 536, 520, 515, 582, 481, 488, 490, 536, 491, 534, 540, 514, 486, 522] payload_rows_per_dst=[5462, 5548, 5572, 5628, 5568, 5570, 5620, 5592] dup_rows_per_dst=[2692, 2608, 2614, 2608, 2592, 2548, 2680, 2634] combine_rows_per_src=[5462, 5548, 5572, 5628, 5568, 5570, 5620, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38792 dup_rows=21028 dup_rows_remote=18262 dedup_ratio=0.320862 router_tokens_per_expert=[520, 498, 518, 527, 512, 488, 532, 489, 492, 509, 526, 549, 525, 498, 495, 523, 515, 522, 558, 481, 492, 554, 479, 529, 528, 457, 509, 529, 544, 517, 537, 556, 492, 494, 540, 499, 508, 496, 518, 498, 528, 523, 472, 524, 518, 457, 503, 482, 504, 527, 495, 512, 522, 460, 536, 516, 528, 488, 515, 502, 544, 529, 505, 525] payload_rows_per_dst=[5588, 5606, 5522, 5510, 5554, 5454, 5558, 5716] dup_rows_per_dst=[2580, 2628, 2506, 2628, 2768, 2560, 2592, 2766] combine_rows_per_src=[5588, 5606, 5522, 5510, 5554, 5454, 5558, 5716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38968 dup_rows=21036 dup_rows_remote=18460 dedup_ratio=0.320984 router_tokens_per_expert=[491, 475, 553, 543, 509, 502, 502, 530, 528, 552, 514, 470, 503, 536, 499, 505, 530, 513, 518, 511, 514, 517, 533, 511, 530, 518, 503, 501, 510, 519, 491, 521, 493, 478, 527, 503, 516, 528, 516, 526, 539, 467, 530, 457, 505, 496, 527, 533, 509, 507, 526, 496, 510, 479, 490, 537, 510, 474, 506, 533, 553, 501, 524, 520] payload_rows_per_dst=[5518, 5596, 5606, 5592, 5576, 5532, 5512, 5568] dup_rows_per_dst=[2692, 2618, 2688, 2594, 2598, 2576, 2596, 2674] combine_rows_per_src=[5518, 5596, 5606, 5592, 5576, 5532, 5512, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38944 dup_rows=21092 dup_rows_remote=18476 dedup_ratio=0.321838 router_tokens_per_expert=[505, 497, 515, 515, 557, 488, 463, 480, 519, 509, 515, 500, 530, 519, 486, 554, 481, 528, 480, 509, 502, 518, 528, 524, 513, 530, 517, 549, 436, 512, 518, 513, 525, 503, 498, 497, 538, 481, 537, 489, 503, 533, 510, 479, 550, 476, 585, 504, 512, 488, 521, 509, 489, 547, 543, 525, 527, 507, 494, 502, 502, 532, 509, 543] payload_rows_per_dst=[5500, 5676, 5534, 5506, 5488, 5914, 5540, 5286] dup_rows_per_dst=[2616, 2764, 2466, 2670, 2648, 3038, 2616, 2274] combine_rows_per_src=[5500, 5676, 5534, 5506, 5488, 5914, 5540, 5286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160038912 dispatch_meta_bytes=229896 combine_payload_bytes=160038912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160268808 ingress_bytes=185204736 link_bytes=185204736 token_bytes=320307720 weight_movement_bytes=25165824 total_nvlink_bytes=345473544 weight_to_token_ratio=0.078568 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409194049e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=229400 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159924248 ingress_bytes=235192320 link_bytes=235192320 token_bytes=319619096 weight_movement_bytes=75497472 total_nvlink_bytes=395116568 weight_to_token_ratio=0.236211 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789541805e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38956 dup_rows=21122 dup_rows_remote=18570 dedup_ratio=0.322296 router_tokens_per_expert=[463, 479, 518, 492, 531, 513, 525, 538, 509, 482, 494, 540, 508, 514, 472, 505, 488, 515, 551, 508, 501, 513, 502, 539, 486, 507, 516, 477, 495, 508, 509, 507, 509, 504, 552, 518, 551, 498, 517, 522, 516, 490, 497, 518, 502, 524, 526, 517, 483, 490, 532, 501, 534, 517, 526, 553, 495, 495, 525, 494, 538, 577, 493, 549] payload_rows_per_dst=[5524, 5464, 5636, 5458, 5638, 5534, 5578, 5582] dup_rows_per_dst=[2594, 2584, 2598, 2552, 2704, 2646, 2694, 2750] combine_rows_per_src=[5524, 5464, 5636, 5458, 5638, 5534, 5578, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158892032 dispatch_meta_bytes=228216 combine_payload_bytes=158892032 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159120248 ingress_bytes=209223680 link_bytes=209223680 token_bytes=318012280 weight_movement_bytes=50331648 total_nvlink_bytes=368343928 weight_to_token_ratio=0.158270 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591950460e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159612928 dispatch_meta_bytes=229712 combine_payload_bytes=159612928 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159842640 ingress_bytes=184778752 link_bytes=184778752 token_bytes=319455568 weight_movement_bytes=25165824 total_nvlink_bytes=344621392 weight_to_token_ratio=0.078777 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405952803e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159514624 dispatch_meta_bytes=229680 combine_payload_bytes=159514624 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159744304 ingress_bytes=184680448 link_bytes=184680448 token_bytes=319258928 weight_movement_bytes=25165824 total_nvlink_bytes=344424752 weight_to_token_ratio=0.078826 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405204823e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44820 payload_rows_remote=39164 dup_rows=20716 dup_rows_remote=18074 dedup_ratio=0.316101 router_tokens_per_expert=[514, 477, 510, 531, 503, 500, 487, 496, 460, 514, 520, 558, 512, 549, 508, 475, 487, 514, 495, 508, 586, 489, 494, 573, 532, 507, 517, 522, 490, 515, 527, 515, 509, 517, 510, 519, 506, 507, 475, 518, 510, 507, 517, 497, 521, 513, 514, 527, 522, 515, 523, 485, 528, 532, 518, 526, 528, 526, 536, 525, 461, 473, 486, 532] payload_rows_per_dst=[5488, 5604, 5628, 5678, 5598, 5596, 5656, 5572] dup_rows_per_dst=[2548, 2588, 2664, 2572, 2524, 2616, 2642, 2562] combine_rows_per_src=[5488, 5604, 5628, 5678, 5598, 5596, 5656, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38826 dup_rows=21116 dup_rows_remote=18494 dedup_ratio=0.322205 router_tokens_per_expert=[497, 474, 529, 483, 541, 543, 494, 522, 505, 504, 521, 500, 463, 503, 471, 566, 515, 522, 491, 527, 500, 511, 485, 502, 513, 531, 511, 541, 506, 512, 560, 504, 502, 558, 543, 509, 488, 517, 477, 514, 513, 521, 501, 536, 493, 506, 474, 506, 542, 510, 511, 521, 530, 504, 542, 522, 513, 508, 505, 523, 504, 543, 478, 507] payload_rows_per_dst=[5546, 5530, 5500, 5640, 5594, 5464, 5578, 5568] dup_rows_per_dst=[2620, 2536, 2606, 2716, 2622, 2636, 2786, 2594] combine_rows_per_src=[5546, 5530, 5500, 5640, 5594, 5464, 5578, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=230104 combine_payload_bytes=159563776 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159793880 ingress_bytes=235061248 link_bytes=235061248 token_bytes=319357656 weight_movement_bytes=75497472 total_nvlink_bytes=394855128 weight_to_token_ratio=0.236404 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788544499e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44544 payload_rows_remote=38894 dup_rows=20992 dup_rows_remote=18264 dedup_ratio=0.320312 router_tokens_per_expert=[489, 529, 466, 519, 515, 509, 524, 510, 524, 469, 537, 569, 528, 559, 484, 519, 482, 533, 509, 510, 505, 489, 464, 549, 501, 536, 521, 546, 507, 492, 524, 527, 490, 503, 481, 534, 508, 506, 533, 532, 518, 503, 517, 517, 510, 507, 490, 511, 470, 491, 465, 495, 516, 500, 538, 533, 543, 487, 537, 522, 498, 522, 551, 495] payload_rows_per_dst=[5498, 5650, 5490, 5624, 5578, 5570, 5544, 5590] dup_rows_per_dst=[2624, 2728, 2592, 2684, 2596, 2576, 2472, 2720] combine_rows_per_src=[5498, 5650, 5490, 5624, 5578, 5570, 5544, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160415744 dispatch_meta_bytes=228952 combine_payload_bytes=160415744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160644696 ingress_bytes=185581568 link_bytes=185581568 token_bytes=321060440 weight_movement_bytes=25165824 total_nvlink_bytes=346226264 weight_to_token_ratio=0.078383 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412061305e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159031296 dispatch_meta_bytes=229280 combine_payload_bytes=159031296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159260576 ingress_bytes=184197120 link_bytes=184197120 token_bytes=318291872 weight_movement_bytes=25165824 total_nvlink_bytes=343457696 weight_to_token_ratio=0.079065 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401527255e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38780 dup_rows=21058 dup_rows_remote=18366 dedup_ratio=0.321320 router_tokens_per_expert=[474, 497, 513, 524, 528, 504, 523, 546, 487, 503, 481, 560, 508, 562, 505, 542, 547, 520, 484, 523, 545, 545, 507, 524, 500, 506, 520, 501, 507, 500, 559, 503, 538, 489, 517, 510, 507, 492, 504, 544, 522, 528, 500, 506, 502, 519, 514, 505, 520, 493, 524, 474, 468, 484, 529, 522, 516, 546, 468, 498, 491, 496, 486, 508] payload_rows_per_dst=[5558, 5532, 5698, 5542, 5570, 5656, 5500, 5422] dup_rows_per_dst=[2660, 2764, 2692, 2650, 2632, 2536, 2528, 2596] combine_rows_per_src=[5558, 5532, 5698, 5542, 5570, 5656, 5500, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=39066 dup_rows=20976 dup_rows_remote=18404 dedup_ratio=0.320068 router_tokens_per_expert=[534, 509, 511, 516, 500, 476, 495, 514, 568, 510, 556, 538, 546, 491, 509, 504, 543, 488, 506, 487, 518, 543, 488, 493, 519, 514, 546, 520, 517, 521, 515, 501, 511, 491, 489, 496, 517, 489, 512, 526, 525, 496, 516, 527, 474, 510, 494, 519, 540, 531, 538, 500, 486, 558, 491, 504, 497, 503, 512, 511, 520, 522, 460, 507] payload_rows_per_dst=[5568, 5682, 5472, 5548, 5566, 5630, 5600, 5494] dup_rows_per_dst=[2542, 2762, 2464, 2710, 2588, 2642, 2696, 2572] combine_rows_per_src=[5568, 5682, 5472, 5548, 5566, 5630, 5600, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38924 dup_rows=21052 dup_rows_remote=18412 dedup_ratio=0.321228 router_tokens_per_expert=[492, 496, 519, 528, 487, 520, 525, 499, 479, 487, 503, 533, 523, 487, 540, 521, 506, 508, 508, 520, 553, 517, 509, 535, 572, 508, 502, 530, 482, 520, 478, 518, 503, 524, 479, 504, 511, 516, 470, 526, 521, 535, 489, 542, 501, 496, 510, 506, 506, 516, 504, 503, 525, 489, 483, 533, 555, 499, 522, 521, 551, 503, 491, 529] payload_rows_per_dst=[5540, 5544, 5594, 5562, 5506, 5560, 5502, 5676] dup_rows_per_dst=[2592, 2602, 2718, 2658, 2560, 2640, 2616, 2666] combine_rows_per_src=[5540, 5544, 5594, 5562, 5506, 5560, 5502, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=228632 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159538456 ingress_bytes=209641472 link_bytes=209641472 token_bytes=318848280 weight_movement_bytes=50331648 total_nvlink_bytes=369179928 weight_to_token_ratio=0.157855 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595129375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44650 payload_rows_remote=38900 dup_rows=20886 dup_rows_remote=18002 dedup_ratio=0.318695 router_tokens_per_expert=[528, 538, 517, 505, 529, 524, 505, 482, 531, 553, 540, 503, 491, 496, 470, 510, 534, 539, 469, 518, 529, 503, 511, 489, 498, 495, 512, 508, 486, 523, 483, 544, 539, 526, 536, 513, 522, 495, 531, 468, 534, 473, 507, 497, 534, 524, 501, 547, 532, 473, 557, 476, 494, 537, 507, 492, 512, 534, 513, 525, 495, 500, 516, 495] payload_rows_per_dst=[5750, 5474, 5600, 5480, 5608, 5634, 5512, 5592] dup_rows_per_dst=[2884, 2492, 2584, 2462, 2652, 2600, 2632, 2580] combine_rows_per_src=[5750, 5474, 5600, 5480, 5608, 5634, 5512, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38880 dup_rows=21088 dup_rows_remote=18446 dedup_ratio=0.321777 router_tokens_per_expert=[503, 497, 514, 528, 507, 505, 476, 501, 519, 509, 527, 493, 471, 490, 537, 548, 539, 480, 531, 520, 524, 489, 537, 507, 509, 514, 510, 523, 548, 491, 499, 511, 546, 490, 543, 489, 504, 501, 503, 503, 496, 526, 507, 508, 533, 496, 506, 519, 499, 528, 520, 516, 511, 517, 507, 530, 495, 491, 484, 520, 544, 528, 520, 531] payload_rows_per_dst=[5514, 5554, 5604, 5568, 5524, 5636, 5474, 5574] dup_rows_per_dst=[2548, 2634, 2650, 2642, 2634, 2546, 2782, 2652] combine_rows_per_src=[5514, 5554, 5604, 5568, 5524, 5636, 5474, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=228584 combine_payload_bytes=158842880 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159071464 ingress_bytes=234340352 link_bytes=234340352 token_bytes=317914344 weight_movement_bytes=75497472 total_nvlink_bytes=393411816 weight_to_token_ratio=0.237477 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783059313e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=229880 combine_payload_bytes=160014336 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160244216 ingress_bytes=210345984 link_bytes=210345984 token_bytes=320258552 weight_movement_bytes=50331648 total_nvlink_bytes=370590200 weight_to_token_ratio=0.157159 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600489897e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=229344 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159662048 ingress_bytes=184598528 link_bytes=184598528 token_bytes=319094752 weight_movement_bytes=25165824 total_nvlink_bytes=344260576 weight_to_token_ratio=0.078866 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404581506e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=227608 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159562008 ingress_bytes=184500224 link_bytes=184500224 token_bytes=318896408 weight_movement_bytes=25165824 total_nvlink_bytes=344062232 weight_to_token_ratio=0.078915 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403833526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44586 payload_rows_remote=39078 dup_rows=20950 dup_rows_remote=18410 dedup_ratio=0.319672 router_tokens_per_expert=[544, 466, 502, 530, 502, 491, 517, 521, 508, 560, 530, 494, 551, 536, 496, 531, 490, 505, 489, 522, 496, 503, 534, 531, 492, 526, 501, 539, 504, 526, 536, 512, 511, 513, 483, 488, 563, 497, 489, 513, 491, 512, 550, 519, 509, 510, 542, 523, 528, 480, 494, 496, 508, 505, 516, 497, 493, 482, 514, 542, 475, 518, 532, 490] payload_rows_per_dst=[5550, 5664, 5562, 5600, 5552, 5614, 5508, 5536] dup_rows_per_dst=[2596, 2748, 2578, 2672, 2562, 2698, 2540, 2556] combine_rows_per_src=[5550, 5664, 5562, 5600, 5552, 5614, 5508, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38912 dup_rows=21068 dup_rows_remote=18472 dedup_ratio=0.321472 router_tokens_per_expert=[494, 500, 495, 501, 535, 509, 516, 512, 525, 495, 511, 525, 495, 494, 527, 461, 530, 519, 547, 511, 524, 511, 513, 546, 507, 512, 492, 481, 518, 505, 554, 534, 511, 497, 520, 469, 512, 506, 513, 548, 517, 508, 511, 523, 539, 493, 536, 513, 538, 514, 500, 544, 502, 520, 495, 471, 558, 453, 506, 534, 527, 500, 486, 505] payload_rows_per_dst=[5542, 5508, 5630, 5620, 5556, 5486, 5602, 5524] dup_rows_per_dst=[2582, 2558, 2772, 2586, 2596, 2794, 2566, 2614] combine_rows_per_src=[5542, 5508, 5630, 5620, 5556, 5486, 5602, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159252480 dispatch_meta_bytes=229304 combine_payload_bytes=159252480 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159481784 ingress_bytes=209584128 link_bytes=209584128 token_bytes=318734264 weight_movement_bytes=50331648 total_nvlink_bytes=369065912 weight_to_token_ratio=0.157911 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594693053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44552 payload_rows_remote=38942 dup_rows=20984 dup_rows_remote=18218 dedup_ratio=0.320190 router_tokens_per_expert=[534, 489, 501, 521, 497, 505, 498, 533, 506, 484, 533, 551, 552, 537, 519, 506, 507, 510, 496, 478, 537, 545, 541, 515, 508, 530, 551, 547, 487, 537, 498, 498, 538, 505, 512, 476, 500, 516, 496, 520, 465, 504, 504, 531, 514, 494, 541, 508, 508, 495, 492, 529, 534, 460, 500, 515, 536, 499, 517, 470, 509, 527, 480, 522] payload_rows_per_dst=[5588, 5610, 5636, 5584, 5548, 5486, 5542, 5558] dup_rows_per_dst=[2568, 2766, 2622, 2728, 2578, 2636, 2524, 2562] combine_rows_per_src=[5588, 5610, 5636, 5584, 5548, 5486, 5542, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=229952 combine_payload_bytes=160063488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160293440 ingress_bytes=185229312 link_bytes=185229312 token_bytes=320356928 weight_movement_bytes=25165824 total_nvlink_bytes=345522752 weight_to_token_ratio=0.078556 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409381044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159383552 dispatch_meta_bytes=229536 combine_payload_bytes=159383552 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159613088 ingress_bytes=234881024 link_bytes=234881024 token_bytes=318996640 weight_movement_bytes=75497472 total_nvlink_bytes=394494112 weight_to_token_ratio=0.236672 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787173203e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=38984 dup_rows=21096 dup_rows_remote=18608 dedup_ratio=0.321899 router_tokens_per_expert=[510, 503, 529, 483, 520, 522, 492, 483, 514, 519, 546, 472, 518, 523, 491, 497, 481, 519, 467, 477, 485, 503, 521, 519, 498, 493, 476, 507, 547, 472, 525, 525, 541, 492, 551, 511, 524, 536, 520, 528, 535, 514, 503, 493, 547, 532, 491, 515, 531, 523, 481, 503, 500, 497, 531, 568, 538, 512, 543, 494, 514, 503, 529, 531] payload_rows_per_dst=[5468, 5570, 5456, 5518, 5626, 5632, 5512, 5658] dup_rows_per_dst=[2616, 2590, 2488, 2568, 2780, 2628, 2756, 2670] combine_rows_per_src=[5468, 5570, 5456, 5518, 5626, 5632, 5512, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44308 payload_rows_remote=38682 dup_rows=21228 dup_rows_remote=18418 dedup_ratio=0.323914 router_tokens_per_expert=[514, 514, 537, 515, 520, 499, 482, 523, 554, 531, 479, 488, 487, 504, 542, 520, 536, 505, 507, 512, 550, 493, 512, 517, 506, 493, 479, 509, 525, 492, 526, 502, 516, 508, 523, 460, 529, 568, 561, 495, 526, 508, 509, 488, 533, 497, 521, 475, 483, 519, 512, 518, 522, 522, 495, 519, 479, 506, 541, 539, 499, 504, 491, 529] payload_rows_per_dst=[5538, 5492, 5462, 5496, 5546, 5554, 5594, 5626] dup_rows_per_dst=[2670, 2538, 2542, 2566, 2660, 2676, 2766, 2810] combine_rows_per_src=[5538, 5492, 5462, 5496, 5546, 5554, 5594, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=228640 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159735072 ingress_bytes=235003904 link_bytes=235003904 token_bytes=319241504 weight_movement_bytes=75497472 total_nvlink_bytes=394738976 weight_to_token_ratio=0.236490 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788108177e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44598 payload_rows_remote=39102 dup_rows=20938 dup_rows_remote=18366 dedup_ratio=0.319489 router_tokens_per_expert=[504, 543, 509, 512, 530, 543, 508, 492, 533, 525, 535, 514, 523, 498, 512, 524, 481, 523, 506, 499, 530, 518, 515, 518, 478, 531, 521, 515, 525, 520, 518, 530, 530, 499, 491, 513, 481, 541, 531, 539, 502, 515, 494, 514, 489, 497, 501, 522, 511, 468, 519, 487, 507, 524, 527, 502, 509, 507, 509, 502, 507, 516, 497, 484] payload_rows_per_dst=[5592, 5684, 5542, 5612, 5648, 5496, 5550, 5474] dup_rows_per_dst=[2690, 2644, 2638, 2664, 2602, 2572, 2540, 2588] combine_rows_per_src=[5592, 5684, 5542, 5612, 5648, 5496, 5550, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44570 payload_rows_remote=38894 dup_rows=20966 dup_rows_remote=18168 dedup_ratio=0.319916 router_tokens_per_expert=[495, 541, 476, 507, 486, 502, 535, 530, 516, 525, 498, 511, 531, 520, 513, 538, 488, 514, 520, 505, 572, 486, 526, 540, 515, 497, 522, 500, 488, 485, 538, 512, 523, 507, 496, 492, 504, 518, 523, 525, 518, 476, 528, 540, 484, 521, 484, 490, 491, 513, 540, 527, 525, 542, 510, 492, 493, 518, 523, 511, 475, 523, 508, 516] payload_rows_per_dst=[5676, 5468, 5570, 5562, 5580, 5506, 5644, 5564] dup_rows_per_dst=[2798, 2508, 2784, 2550, 2544, 2576, 2636, 2570] combine_rows_per_src=[5676, 5468, 5570, 5562, 5580, 5506, 5644, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44686 payload_rows_remote=39116 dup_rows=20850 dup_rows_remote=18318 dedup_ratio=0.318146 router_tokens_per_expert=[489, 492, 494, 563, 500, 466, 537, 531, 514, 538, 498, 533, 515, 468, 495, 516, 521, 517, 543, 503, 482, 525, 494, 530, 522, 503, 491, 493, 499, 520, 536, 487, 462, 501, 552, 517, 505, 534, 521, 496, 497, 526, 497, 547, 492, 515, 528, 521, 511, 510, 505, 517, 504, 525, 479, 529, 533, 529, 509, 533, 524, 502, 513, 519] payload_rows_per_dst=[5578, 5580, 5606, 5570, 5578, 5556, 5590, 5628] dup_rows_per_dst=[2566, 2574, 2624, 2532, 2598, 2690, 2570, 2696] combine_rows_per_src=[5578, 5580, 5606, 5570, 5578, 5556, 5590, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159678464 dispatch_meta_bytes=230368 combine_payload_bytes=159678464 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159908832 ingress_bytes=210010112 link_bytes=210010112 token_bytes=319587296 weight_movement_bytes=50331648 total_nvlink_bytes=369918944 weight_to_token_ratio=0.157490 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597934299e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158441472 dispatch_meta_bytes=228400 combine_payload_bytes=158441472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158669872 ingress_bytes=183607296 link_bytes=183607296 token_bytes=317111344 weight_movement_bytes=25165824 total_nvlink_bytes=342277168 weight_to_token_ratio=0.079360 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397039376e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160161792 dispatch_meta_bytes=229872 combine_payload_bytes=160161792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160391664 ingress_bytes=185327616 link_bytes=185327616 token_bytes=320553456 weight_movement_bytes=25165824 total_nvlink_bytes=345719280 weight_to_token_ratio=0.078507 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410129024e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=228248 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159538072 ingress_bytes=184475648 link_bytes=184475648 token_bytes=318847896 weight_movement_bytes=25165824 total_nvlink_bytes=344013720 weight_to_token_ratio=0.078927 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403646531e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44430 payload_rows_remote=38830 dup_rows=21106 dup_rows_remote=18448 dedup_ratio=0.322052 router_tokens_per_expert=[503, 482, 516, 536, 524, 566, 507, 498, 514, 542, 530, 501, 505, 522, 478, 488, 499, 480, 539, 505, 507, 504, 567, 510, 503, 521, 509, 496, 506, 533, 522, 494, 558, 540, 492, 546, 488, 508, 484, 490, 528, 521, 498, 531, 471, 515, 485, 507, 534, 538, 490, 498, 515, 542, 496, 516, 507, 530, 519, 445, 552, 504, 489, 524] payload_rows_per_dst=[5570, 5528, 5540, 5566, 5572, 5516, 5600, 5538] dup_rows_per_dst=[2694, 2632, 2682, 2602, 2640, 2596, 2658, 2602] combine_rows_per_src=[5570, 5528, 5540, 5566, 5572, 5516, 5600, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44462 payload_rows_remote=38842 dup_rows=21074 dup_rows_remote=18406 dedup_ratio=0.321564 router_tokens_per_expert=[545, 503, 488, 523, 553, 526, 488, 536, 512, 507, 546, 503, 507, 511, 524, 467, 502, 513, 477, 520, 555, 486, 506, 512, 546, 465, 537, 481, 522, 551, 522, 528, 501, 509, 505, 515, 534, 526, 552, 502, 537, 513, 501, 509, 529, 517, 503, 471, 519, 480, 504, 516, 495, 492, 511, 502, 515, 530, 509, 491, 496, 529, 479, 514] payload_rows_per_dst=[5600, 5514, 5548, 5560, 5620, 5580, 5502, 5538] dup_rows_per_dst=[2724, 2640, 2594, 2744, 2668, 2580, 2536, 2588] combine_rows_per_src=[5600, 5514, 5548, 5560, 5620, 5580, 5502, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=160219136 dispatch_meta_bytes=229736 combine_payload_bytes=160219136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160448872 ingress_bytes=185384960 link_bytes=185384960 token_bytes=320668008 weight_movement_bytes=25165824 total_nvlink_bytes=345833832 weight_to_token_ratio=0.078479 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410565345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44514 payload_rows_remote=39018 dup_rows=21022 dup_rows_remote=18346 dedup_ratio=0.320770 router_tokens_per_expert=[545, 522, 497, 540, 477, 531, 489, 512, 513, 519, 494, 504, 507, 489, 541, 519, 524, 494, 506, 517, 529, 487, 509, 545, 524, 506, 541, 538, 496, 498, 485, 515, 500, 462, 483, 547, 532, 528, 475, 541, 536, 495, 528, 533, 522, 478, 486, 475, 485, 494, 539, 514, 522, 485, 531, 515, 514, 536, 511, 521, 555, 478, 528, 506] payload_rows_per_dst=[5590, 5496, 5584, 5500, 5556, 5566, 5600, 5622] dup_rows_per_dst=[2636, 2676, 2638, 2706, 2580, 2540, 2570, 2676] combine_rows_per_src=[5590, 5496, 5584, 5500, 5556, 5566, 5600, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159047680 dispatch_meta_bytes=229112 combine_payload_bytes=159047680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159276792 ingress_bytes=184213504 link_bytes=184213504 token_bytes=318324472 weight_movement_bytes=25165824 total_nvlink_bytes=343490296 weight_to_token_ratio=0.079057 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401651918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159096832 dispatch_meta_bytes=228992 combine_payload_bytes=159096832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159325824 ingress_bytes=209428480 link_bytes=209428480 token_bytes=318422656 weight_movement_bytes=50331648 total_nvlink_bytes=368754304 weight_to_token_ratio=0.158066 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593508752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=39018 dup_rows=21012 dup_rows_remote=18450 dedup_ratio=0.320618 router_tokens_per_expert=[504, 532, 534, 515, 488, 541, 519, 546, 505, 520, 497, 523, 514, 526, 544, 515, 513, 529, 479, 522, 506, 514, 451, 520, 518, 484, 484, 487, 519, 514, 492, 562, 508, 527, 491, 524, 496, 518, 523, 528, 498, 505, 498, 492, 546, 537, 495, 542, 517, 514, 506, 527, 482, 524, 500, 506, 512, 498, 487, 495, 493, 516, 511, 535] payload_rows_per_dst=[5656, 5630, 5506, 5480, 5586, 5604, 5572, 5490] dup_rows_per_dst=[2702, 2658, 2562, 2640, 2644, 2622, 2580, 2604] combine_rows_per_src=[5656, 5630, 5506, 5480, 5586, 5604, 5572, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38812 dup_rows=21114 dup_rows_remote=18504 dedup_ratio=0.322174 router_tokens_per_expert=[541, 567, 494, 544, 510, 497, 487, 510, 515, 549, 529, 558, 513, 501, 487, 500, 496, 524, 499, 493, 542, 488, 556, 474, 501, 525, 529, 516, 485, 505, 491, 521, 512, 499, 514, 490, 550, 518, 524, 466, 514, 512, 512, 516, 527, 511, 511, 523, 524, 508, 512, 514, 484, 467, 517, 509, 467, 506, 508, 556, 486, 523, 524, 517] payload_rows_per_dst=[5334, 5560, 5570, 5522, 5510, 5628, 5688, 5610] dup_rows_per_dst=[2440, 2744, 2574, 2624, 2590, 2624, 2908, 2610] combine_rows_per_src=[5334, 5560, 5570, 5522, 5510, 5628, 5688, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38776 dup_rows=21146 dup_rows_remote=18428 dedup_ratio=0.322662 router_tokens_per_expert=[488, 519, 517, 532, 521, 508, 500, 543, 518, 507, 544, 498, 518, 546, 542, 550, 545, 519, 503, 502, 492, 506, 502, 507, 475, 500, 499, 506, 507, 563, 546, 509, 502, 549, 550, 535, 533, 492, 455, 516, 482, 537, 529, 537, 536, 534, 512, 499, 526, 481, 487, 526, 488, 481, 494, 503, 503, 483, 506, 511, 495, 485, 481, 488] payload_rows_per_dst=[5556, 5646, 5508, 5592, 5590, 5614, 5462, 5422] dup_rows_per_dst=[2700, 2800, 2644, 2618, 2674, 2718, 2510, 2482] combine_rows_per_src=[5556, 5646, 5508, 5592, 5590, 5614, 5462, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=229456 combine_payload_bytes=159817728 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160047184 ingress_bytes=210149376 link_bytes=210149376 token_bytes=319864912 weight_movement_bytes=50331648 total_nvlink_bytes=370196560 weight_to_token_ratio=0.157353 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598993937e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44306 payload_rows_remote=38782 dup_rows=21230 dup_rows_remote=18564 dedup_ratio=0.323944 router_tokens_per_expert=[482, 541, 504, 470, 479, 508, 524, 499, 484, 499, 506, 503, 488, 541, 499, 495, 528, 513, 498, 506, 514, 515, 545, 495, 530, 499, 512, 505, 501, 513, 515, 497, 478, 486, 511, 528, 530, 491, 510, 524, 506, 508, 509, 477, 545, 517, 551, 515, 525, 511, 499, 490, 514, 517, 512, 527, 554, 527, 510, 552, 517, 560, 522, 537] payload_rows_per_dst=[5524, 5398, 5604, 5614, 5492, 5470, 5542, 5662] dup_rows_per_dst=[2666, 2460, 2752, 2666, 2638, 2632, 2648, 2768] combine_rows_per_src=[5524, 5398, 5604, 5614, 5492, 5470, 5542, 5662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44378 payload_rows_remote=38804 dup_rows=21158 dup_rows_remote=18268 dedup_ratio=0.322845 router_tokens_per_expert=[511, 546, 526, 513, 524, 519, 508, 516, 500, 517, 490, 516, 511, 508, 521, 552, 505, 512, 504, 527, 518, 507, 492, 491, 525, 513, 507, 535, 505, 531, 556, 560, 483, 515, 520, 475, 517, 484, 458, 515, 531, 519, 486, 500, 520, 511, 518, 489, 525, 469, 552, 530, 503, 512, 487, 500, 494, 492, 561, 530, 482, 499, 529, 496] payload_rows_per_dst=[5618, 5562, 5534, 5574, 5446, 5578, 5580, 5486] dup_rows_per_dst=[2708, 2668, 2578, 2890, 2488, 2570, 2576, 2680] combine_rows_per_src=[5618, 5562, 5534, 5574, 5446, 5578, 5580, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=229872 combine_payload_bytes=159817728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160047600 ingress_bytes=184983552 link_bytes=184983552 token_bytes=319865328 weight_movement_bytes=25165824 total_nvlink_bytes=345031152 weight_to_token_ratio=0.078676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407511094e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158973952 dispatch_meta_bytes=229264 combine_payload_bytes=158973952 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159203216 ingress_bytes=209305600 link_bytes=209305600 token_bytes=318177168 weight_movement_bytes=50331648 total_nvlink_bytes=368508816 weight_to_token_ratio=0.158187 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592573777e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=228816 combine_payload_bytes=158826496 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159055312 ingress_bytes=234323968 link_bytes=234323968 token_bytes=317881808 weight_movement_bytes=75497472 total_nvlink_bytes=393379280 weight_to_token_ratio=0.237502 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782934650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158851072 dispatch_meta_bytes=229384 combine_payload_bytes=158851072 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159080456 ingress_bytes=209182720 link_bytes=209182720 token_bytes=317931528 weight_movement_bytes=50331648 total_nvlink_bytes=368263176 weight_to_token_ratio=0.158310 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591638802e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=39098 dup_rows=20946 dup_rows_remote=18454 dedup_ratio=0.319611 router_tokens_per_expert=[479, 490, 487, 496, 515, 509, 530, 511, 538, 489, 532, 530, 524, 476, 558, 507, 538, 508, 508, 508, 507, 513, 509, 524, 486, 517, 534, 544, 504, 493, 520, 538, 540, 527, 511, 515, 526, 503, 544, 506, 523, 511, 515, 497, 512, 531, 493, 493, 495, 495, 502, 473, 488, 513, 517, 509, 489, 535, 537, 505, 489, 528, 512, 512] payload_rows_per_dst=[5488, 5596, 5606, 5554, 5620, 5640, 5492, 5594] dup_rows_per_dst=[2546, 2712, 2624, 2718, 2724, 2510, 2492, 2620] combine_rows_per_src=[5488, 5596, 5606, 5554, 5620, 5640, 5492, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38850 dup_rows=21120 dup_rows_remote=18454 dedup_ratio=0.322266 router_tokens_per_expert=[530, 533, 479, 518, 557, 506, 508, 516, 498, 520, 536, 531, 515, 469, 511, 517, 523, 483, 509, 495, 500, 506, 538, 533, 480, 508, 464, 507, 487, 525, 500, 512, 480, 516, 493, 562, 490, 523, 525, 527, 519, 544, 495, 544, 483, 522, 539, 565, 472, 473, 521, 523, 461, 522, 468, 521, 530, 522, 515, 510, 510, 527, 503, 549] payload_rows_per_dst=[5576, 5508, 5590, 5476, 5566, 5684, 5486, 5530] dup_rows_per_dst=[2718, 2686, 2584, 2490, 2666, 2738, 2436, 2802] combine_rows_per_src=[5576, 5508, 5590, 5476, 5566, 5684, 5486, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=228288 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159169472 ingress_bytes=184107008 link_bytes=184107008 token_bytes=318110656 weight_movement_bytes=25165824 total_nvlink_bytes=343276480 weight_to_token_ratio=0.079110 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400841607e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44456 payload_rows_remote=38944 dup_rows=21080 dup_rows_remote=18484 dedup_ratio=0.321655 router_tokens_per_expert=[480, 525, 514, 505, 537, 515, 479, 517, 526, 479, 523, 554, 498, 504, 471, 499, 528, 538, 522, 535, 473, 489, 477, 512, 514, 529, 514, 539, 524, 505, 507, 543, 509, 488, 562, 497, 518, 502, 514, 563, 509, 550, 510, 522, 521, 494, 490, 490, 501, 511, 526, 475, 552, 514, 494, 490, 523, 493, 505, 553, 515, 505, 508, 489] payload_rows_per_dst=[5572, 5512, 5550, 5690, 5586, 5518, 5468, 5560] dup_rows_per_dst=[2572, 2596, 2598, 2660, 2720, 2654, 2658, 2622] combine_rows_per_src=[5572, 5512, 5550, 5690, 5586, 5518, 5468, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160145408 dispatch_meta_bytes=230208 combine_payload_bytes=160145408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160375616 ingress_bytes=185311232 link_bytes=185311232 token_bytes=320521024 weight_movement_bytes=25165824 total_nvlink_bytes=345686848 weight_to_token_ratio=0.078515 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410004360e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159129600 dispatch_meta_bytes=229216 combine_payload_bytes=159129600 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159358816 ingress_bytes=209461248 link_bytes=209461248 token_bytes=318488416 weight_movement_bytes=50331648 total_nvlink_bytes=368820064 weight_to_token_ratio=0.158033 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593758078e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44810 payload_rows_remote=39212 dup_rows=20726 dup_rows_remote=18086 dedup_ratio=0.316254 router_tokens_per_expert=[514, 510, 558, 485, 561, 473, 464, 504, 501, 505, 485, 548, 477, 533, 521, 515, 517, 529, 481, 536, 524, 477, 500, 555, 487, 531, 513, 511, 493, 535, 470, 505, 513, 526, 514, 535, 499, 531, 541, 502, 527, 548, 494, 511, 487, 480, 504, 548, 504, 528, 546, 531, 524, 497, 488, 524, 483, 531, 537, 508, 524, 491, 473, 501] payload_rows_per_dst=[5534, 5606, 5598, 5600, 5622, 5622, 5622, 5606] dup_rows_per_dst=[2604, 2564, 2640, 2490, 2700, 2576, 2662, 2490] combine_rows_per_src=[5534, 5606, 5598, 5600, 5622, 5622, 5622, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38908 dup_rows=21046 dup_rows_remote=18498 dedup_ratio=0.321136 router_tokens_per_expert=[526, 495, 536, 510, 523, 524, 513, 496, 515, 492, 510, 527, 494, 535, 532, 494, 541, 494, 493, 511, 504, 535, 548, 499, 485, 493, 502, 509, 521, 513, 508, 508, 495, 546, 563, 543, 508, 456, 503, 502, 492, 530, 518, 503, 536, 493, 535, 527, 531, 496, 496, 516, 510, 484, 509, 525, 512, 498, 507, 525, 533, 499, 493, 498] payload_rows_per_dst=[5538, 5498, 5540, 5558, 5454, 5606, 5714, 5582] dup_rows_per_dst=[2708, 2644, 2684, 2520, 2456, 2718, 2768, 2548] combine_rows_per_src=[5538, 5498, 5540, 5558, 5454, 5606, 5714, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44400 payload_rows_remote=38882 dup_rows=21136 dup_rows_remote=18516 dedup_ratio=0.322510 router_tokens_per_expert=[508, 483, 501, 496, 502, 521, 530, 516, 526, 532, 510, 491, 493, 513, 548, 480, 510, 504, 513, 539, 502, 519, 520, 510, 533, 496, 489, 476, 509, 481, 514, 487, 517, 503, 517, 498, 512, 559, 498, 495, 499, 484, 527, 502, 498, 489, 513, 557, 500, 523, 558, 545, 543, 520, 513, 525, 538, 495, 509, 524, 509, 518, 505, 523] payload_rows_per_dst=[5510, 5586, 5482, 5450, 5598, 5518, 5656, 5600] dup_rows_per_dst=[2604, 2600, 2752, 2520, 2600, 2620, 2798, 2642] combine_rows_per_src=[5510, 5586, 5482, 5450, 5598, 5518, 5656, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159514624 dispatch_meta_bytes=229712 combine_payload_bytes=159514624 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159744336 ingress_bytes=235012096 link_bytes=235012096 token_bytes=319258960 weight_movement_bytes=75497472 total_nvlink_bytes=394756432 weight_to_token_ratio=0.236477 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788170509e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44622 payload_rows_remote=38978 dup_rows=20914 dup_rows_remote=18318 dedup_ratio=0.319122 router_tokens_per_expert=[567, 509, 495, 476, 520, 511, 508, 476, 532, 502, 485, 505, 548, 524, 514, 539, 489, 502, 537, 516, 492, 555, 497, 506, 481, 493, 516, 534, 510, 494, 514, 483, 500, 561, 527, 516, 513, 525, 512, 491, 511, 503, 540, 504, 529, 507, 530, 516, 487, 525, 517, 510, 511, 543, 486, 485, 526, 518, 469, 533, 483, 504, 532, 524] payload_rows_per_dst=[5644, 5554, 5574, 5646, 5534, 5594, 5516, 5560] dup_rows_per_dst=[2596, 2644, 2602, 2700, 2756, 2390, 2612, 2614] combine_rows_per_src=[5644, 5554, 5574, 5646, 5534, 5594, 5516, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=39038 dup_rows=20988 dup_rows_remote=18548 dedup_ratio=0.320251 router_tokens_per_expert=[508, 495, 525, 534, 519, 507, 533, 492, 521, 518, 547, 570, 538, 477, 504, 484, 510, 503, 526, 494, 539, 478, 542, 465, 517, 489, 508, 490, 477, 538, 487, 469, 487, 488, 545, 524, 483, 538, 508, 501, 524, 509, 500, 540, 495, 518, 536, 499, 508, 542, 514, 509, 517, 525, 522, 515, 526, 500, 517, 502, 518, 533, 517, 504] payload_rows_per_dst=[5588, 5652, 5494, 5510, 5494, 5550, 5612, 5648] dup_rows_per_dst=[2638, 2666, 2620, 2440, 2654, 2692, 2692, 2586] combine_rows_per_src=[5588, 5652, 5494, 5510, 5494, 5550, 5612, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160612352 dispatch_meta_bytes=229192 combine_payload_bytes=160612352 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160841544 ingress_bytes=236109824 link_bytes=236109824 token_bytes=321453896 weight_movement_bytes=75497472 total_nvlink_bytes=396951368 weight_to_token_ratio=0.234863 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.796522951e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159367168 dispatch_meta_bytes=229624 combine_payload_bytes=159367168 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159596792 ingress_bytes=184532992 link_bytes=184532992 token_bytes=318963960 weight_movement_bytes=25165824 total_nvlink_bytes=344129784 weight_to_token_ratio=0.078899 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404082853e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159260672 dispatch_meta_bytes=229592 combine_payload_bytes=159260672 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159490264 ingress_bytes=209592320 link_bytes=209592320 token_bytes=318750936 weight_movement_bytes=50331648 total_nvlink_bytes=369082584 weight_to_token_ratio=0.157903 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594755385e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44592 payload_rows_remote=38958 dup_rows=20944 dup_rows_remote=18228 dedup_ratio=0.319580 router_tokens_per_expert=[490, 517, 525, 493, 535, 523, 503, 503, 497, 566, 513, 477, 506, 537, 500, 511, 497, 494, 496, 505, 513, 510, 517, 502, 492, 500, 526, 486, 510, 475, 516, 489, 528, 492, 529, 528, 523, 563, 515, 484, 500, 545, 514, 498, 516, 505, 522, 511, 493, 585, 573, 464, 536, 518, 491, 515, 509, 513, 501, 510, 497, 534, 508, 524] payload_rows_per_dst=[5542, 5566, 5566, 5474, 5670, 5626, 5634, 5514] dup_rows_per_dst=[2636, 2648, 2502, 2514, 2654, 2596, 2716, 2678] combine_rows_per_src=[5542, 5566, 5566, 5474, 5670, 5626, 5634, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159653888 dispatch_meta_bytes=229184 combine_payload_bytes=159653888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159883072 ingress_bytes=184819712 link_bytes=184819712 token_bytes=319536960 weight_movement_bytes=25165824 total_nvlink_bytes=344702784 weight_to_token_ratio=0.078757 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406264461e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44584 payload_rows_remote=38942 dup_rows=20952 dup_rows_remote=18300 dedup_ratio=0.319702 router_tokens_per_expert=[467, 506, 520, 518, 519, 505, 515, 557, 499, 510, 515, 472, 506, 499, 546, 486, 521, 513, 507, 547, 515, 478, 517, 498, 523, 496, 513, 487, 508, 529, 499, 486, 558, 506, 469, 557, 512, 477, 520, 548, 486, 517, 510, 518, 480, 520, 521, 495, 542, 514, 532, 463, 549, 524, 523, 522, 556, 499, 509, 527, 508, 492, 518, 519] payload_rows_per_dst=[5558, 5524, 5640, 5524, 5642, 5490, 5622, 5584] dup_rows_per_dst=[2656, 2542, 2552, 2558, 2652, 2604, 2716, 2672] combine_rows_per_src=[5558, 5524, 5640, 5524, 5642, 5490, 5622, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159899648 dispatch_meta_bytes=230344 combine_payload_bytes=159899648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160129992 ingress_bytes=185065472 link_bytes=185065472 token_bytes=320029640 weight_movement_bytes=25165824 total_nvlink_bytes=345195464 weight_to_token_ratio=0.078636 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408134411e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44488 payload_rows_remote=39006 dup_rows=21048 dup_rows_remote=18486 dedup_ratio=0.321167 router_tokens_per_expert=[495, 535, 499, 505, 506, 532, 497, 515, 505, 526, 481, 501, 515, 478, 510, 506, 509, 489, 526, 496, 499, 552, 516, 547, 522, 465, 521, 510, 501, 487, 488, 549, 529, 530, 489, 536, 485, 532, 536, 492, 509, 508, 513, 491, 512, 527, 552, 505, 504, 473, 557, 520, 501, 514, 499, 507, 502, 504, 558, 506, 539, 525, 494, 536] payload_rows_per_dst=[5580, 5482, 5644, 5570, 5584, 5552, 5516, 5560] dup_rows_per_dst=[2588, 2562, 2624, 2516, 2674, 2682, 2634, 2768] combine_rows_per_src=[5580, 5482, 5644, 5570, 5584, 5552, 5516, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159571968 dispatch_meta_bytes=228744 combine_payload_bytes=159571968 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159800712 ingress_bytes=209903616 link_bytes=209903616 token_bytes=319372680 weight_movement_bytes=50331648 total_nvlink_bytes=369704328 weight_to_token_ratio=0.157595 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597123988e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=228968 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159735400 ingress_bytes=209838080 link_bytes=209838080 token_bytes=319241832 weight_movement_bytes=50331648 total_nvlink_bytes=369573480 weight_to_token_ratio=0.157660 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596625334e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5113171184 prefetch_bytes=327155712 reduce_bytes=301989888 weight_movement_bytes=629145600 egress_bytes=2558422256 ingress_bytes=3183894528 link_bytes=3183894528 total_nvlink_bytes=5742316784 weight_to_token_ratio=0.123044 prefetch_share_of_ingress=0.102753 dedup_ratio=0.320759 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=12 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.514109911e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44710 payload_rows_remote=39044 dup_rows=20826 dup_rows_remote=18188 dedup_ratio=0.317780 router_tokens_per_expert=[493, 516, 488, 514, 512, 512, 527, 535, 539, 500, 533, 465, 542, 499, 503, 524, 528, 495, 523, 537, 497, 555, 494, 523, 473, 522, 518, 498, 474, 507, 506, 496, 529, 493, 507, 531, 559, 517, 513, 516, 564, 490, 517, 537, 529, 498, 493, 516, 473, 482, 492, 504, 543, 519, 483, 505, 512, 523, 517, 483, 513, 521, 542, 499] payload_rows_per_dst=[5614, 5590, 5666, 5466, 5600, 5624, 5472, 5678] dup_rows_per_dst=[2580, 2620, 2638, 2522, 2730, 2664, 2530, 2542] combine_rows_per_src=[5614, 5590, 5666, 5466, 5600, 5624, 5472, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44324 payload_rows_remote=38870 dup_rows=21212 dup_rows_remote=18582 dedup_ratio=0.323669 router_tokens_per_expert=[518, 522, 521, 526, 487, 501, 518, 501, 518, 485, 478, 547, 501, 506, 540, 522, 525, 551, 473, 513, 490, 522, 540, 506, 512, 517, 509, 510, 470, 536, 516, 527, 538, 521, 474, 510, 563, 504, 494, 472, 525, 525, 505, 508, 508, 503, 530, 549, 509, 486, 520, 511, 512, 489, 531, 539, 514, 471, 524, 523, 519, 490, 498, 495] payload_rows_per_dst=[5614, 5364, 5528, 5532, 5662, 5612, 5558, 5454] dup_rows_per_dst=[2574, 2420, 2712, 2662, 2900, 2694, 2620, 2630] combine_rows_per_src=[5614, 5364, 5528, 5532, 5662, 5612, 5558, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159768576 dispatch_meta_bytes=229968 combine_payload_bytes=159768576 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159998544 ingress_bytes=210100224 link_bytes=210100224 token_bytes=319767120 weight_movement_bytes=50331648 total_nvlink_bytes=370098768 weight_to_token_ratio=0.157401 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598619947e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44352 payload_rows_remote=38758 dup_rows=21184 dup_rows_remote=18488 dedup_ratio=0.323242 router_tokens_per_expert=[517, 540, 509, 495, 499, 517, 493, 504, 533, 514, 530, 525, 500, 538, 562, 545, 535, 502, 484, 502, 481, 521, 515, 540, 516, 515, 518, 514, 495, 507, 511, 500, 516, 500, 514, 471, 495, 505, 547, 528, 489, 512, 523, 519, 534, 504, 502, 562, 559, 459, 456, 484, 500, 537, 462, 503, 515, 510, 519, 530, 513, 522, 494, 507] payload_rows_per_dst=[5494, 5714, 5520, 5538, 5448, 5594, 5464, 5580] dup_rows_per_dst=[2654, 2780, 2640, 2614, 2704, 2696, 2456, 2640] combine_rows_per_src=[5494, 5714, 5520, 5538, 5448, 5594, 5464, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5109954856 prefetch_bytes=327155712 reduce_bytes=427819008 weight_movement_bytes=754974720 egress_bytes=2556811560 ingress_bytes=3308118016 link_bytes=3308118016 total_nvlink_bytes=5864929576 weight_to_token_ratio=0.147746 prefetch_share_of_ingress=0.098895 dedup_ratio=0.320492 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=17 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.573184736e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5103108208 prefetch_bytes=402653184 reduce_bytes=226492416 weight_movement_bytes=629145600 egress_bytes=2553389168 ingress_bytes=3178864640 link_bytes=3178864640 total_nvlink_bytes=5732253808 weight_to_token_ratio=0.123287 prefetch_share_of_ingress=0.126666 dedup_ratio=0.321653 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=9 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042509 gemm_inner_pad_upper_bound=20320 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.511717933e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38922 dup_rows=21026 dup_rows_remote=18394 dedup_ratio=0.320831 router_tokens_per_expert=[488, 481, 492, 472, 499, 529, 516, 496, 509, 543, 507, 535, 506, 482, 545, 523, 503, 504, 518, 506, 523, 540, 508, 535, 529, 550, 457, 508, 562, 491, 489, 484, 528, 507, 501, 528, 551, 521, 507, 569, 494, 500, 482, 509, 510, 484, 536, 490, 531, 487, 507, 535, 535, 546, 524, 493, 510, 508, 491, 524, 498, 496, 521, 515] payload_rows_per_dst=[5588, 5622, 5586, 5914, 5262, 5500, 5522, 5516] dup_rows_per_dst=[2632, 2784, 2604, 3182, 2206, 2360, 2772, 2486] combine_rows_per_src=[5588, 5622, 5586, 5914, 5262, 5500, 5522, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159924224 dispatch_meta_bytes=228928 combine_payload_bytes=159924224 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160153152 ingress_bytes=210255872 link_bytes=210255872 token_bytes=320077376 weight_movement_bytes=50331648 total_nvlink_bytes=370409024 weight_to_token_ratio=0.157248 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599804249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5111760160 prefetch_bytes=327155712 reduce_bytes=402653184 weight_movement_bytes=729808896 egress_bytes=2557715744 ingress_bytes=3283853312 link_bytes=3283853312 total_nvlink_bytes=5841569056 weight_to_token_ratio=0.142771 prefetch_share_of_ingress=0.099626 dedup_ratio=0.320543 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=16 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.561645589e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159211520 dispatch_meta_bytes=229808 combine_payload_bytes=159211520 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159441328 ingress_bytes=209543168 link_bytes=209543168 token_bytes=318652848 weight_movement_bytes=50331648 total_nvlink_bytes=368984496 weight_to_token_ratio=0.157951 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594381395e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158752768 dispatch_meta_bytes=228984 combine_payload_bytes=158752768 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158981752 ingress_bytes=209084416 link_bytes=209084416 token_bytes=317734520 weight_movement_bytes=50331648 total_nvlink_bytes=368066168 weight_to_token_ratio=0.158408 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590890822e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=229264 combine_payload_bytes=159424512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159653776 ingress_bytes=184590336 link_bytes=184590336 token_bytes=319078288 weight_movement_bytes=25165824 total_nvlink_bytes=344244112 weight_to_token_ratio=0.078870 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404519175e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5111594480 prefetch_bytes=327155712 reduce_bytes=503316480 weight_movement_bytes=830472192 egress_bytes=2557631984 ingress_bytes=3384434688 link_bytes=3384434688 total_nvlink_bytes=5942066672 weight_to_token_ratio=0.162468 prefetch_share_of_ingress=0.096665 dedup_ratio=0.320292 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=20 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.609477342e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5095106664 prefetch_bytes=352321536 reduce_bytes=402653184 weight_movement_bytes=754974720 egress_bytes=2549385320 ingress_bytes=3300696064 link_bytes=3300696064 total_nvlink_bytes=5850081384 weight_to_token_ratio=0.148176 prefetch_share_of_ingress=0.106742 dedup_ratio=0.321693 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=16 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.569655206e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5110368512 prefetch_bytes=352321536 reduce_bytes=226492416 weight_movement_bytes=578813952 egress_bytes=2557020416 ingress_bytes=3132162048 link_bytes=3132162048 total_nvlink_bytes=5689182464 weight_to_token_ratio=0.113263 prefetch_share_of_ingress=0.112485 dedup_ratio=0.320864 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=9 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.489508386e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5102115864 prefetch_bytes=377487360 reduce_bytes=301989888 weight_movement_bytes=679477248 egress_bytes=2552888344 ingress_bytes=3228704768 link_bytes=3228704768 total_nvlink_bytes=5781593112 weight_to_token_ratio=0.133176 prefetch_share_of_ingress=0.116916 dedup_ratio=0.320892 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=12 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.535419545e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none + [2026-09-21 08:32:49] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 211.1 | throughput per GPU (TFLOP/s/GPU): 152.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.685543E+00 | loss scale: 1.0 | grad norm: 8.225 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44544 payload_rows_remote=38896 dup_rows=20992 dup_rows_remote=18370 dedup_ratio=0.320312 router_tokens_per_expert=[502, 530, 531, 514, 537, 527, 513, 508, 540, 500, 518, 505, 537, 516, 499, 502, 488, 507, 542, 543, 508, 529, 522, 491, 495, 477, 532, 453, 525, 511, 470, 526, 520, 506, 504, 498, 534, 522, 524, 503, 497, 494, 478, 535, 527, 505, 498, 503, 530, 551, 492, 511, 501, 521, 536, 493, 563, 500, 517, 519, 490, 508, 515, 475] payload_rows_per_dst=[5602, 5548, 5588, 5470, 5554, 5554, 5648, 5580] dup_rows_per_dst=[2722, 2686, 2672, 2508, 2668, 2520, 2622, 2594] combine_rows_per_src=[5602, 5548, 5588, 5470, 5554, 5554, 5648, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38830 dup_rows=21122 dup_rows_remote=18534 dedup_ratio=0.322296 router_tokens_per_expert=[504, 529, 497, 515, 522, 505, 509, 549, 511, 542, 508, 511, 535, 518, 518, 547, 515, 521, 493, 508, 524, 516, 524, 485, 503, 522, 506, 505, 527, 524, 474, 481, 512, 531, 509, 504, 475, 556, 477, 520, 505, 507, 494, 482, 499, 464, 503, 513, 503, 515, 530, 552, 505, 520, 565, 473, 479, 510, 480, 487, 535, 539, 514, 562] payload_rows_per_dst=[5580, 5618, 5584, 5440, 5568, 5412, 5630, 5582] dup_rows_per_dst=[2680, 2762, 2588, 2644, 2600, 2522, 2696, 2630] combine_rows_per_src=[5580, 5618, 5584, 5440, 5568, 5412, 5630, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=38958 dup_rows=20910 dup_rows_remote=18262 dedup_ratio=0.319061 router_tokens_per_expert=[523, 486, 465, 509, 505, 535, 489, 519, 503, 512, 483, 530, 490, 525, 510, 498, 521, 523, 505, 502, 536, 484, 498, 503, 499, 513, 523, 544, 500, 509, 568, 531, 520, 489, 509, 494, 508, 510, 548, 512, 547, 562, 524, 525, 514, 463, 538, 510, 517, 507, 520, 500, 493, 481, 522, 499, 527, 501, 517, 495, 511, 510, 539, 515] payload_rows_per_dst=[5352, 5596, 5296, 5756, 5674, 5742, 5542, 5668] dup_rows_per_dst=[2420, 2626, 2244, 2978, 2730, 2708, 2556, 2648] combine_rows_per_src=[5352, 5596, 5296, 5756, 5674, 5742, 5542, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38900 dup_rows=21076 dup_rows_remote=18538 dedup_ratio=0.321594 router_tokens_per_expert=[503, 530, 509, 465, 493, 513, 509, 534, 549, 530, 547, 501, 493, 510, 503, 482, 527, 529, 535, 472, 503, 508, 501, 555, 480, 504, 528, 508, 537, 498, 503, 510, 525, 479, 537, 509, 469, 500, 482, 548, 490, 513, 502, 492, 516, 516, 523, 535, 515, 501, 515, 503, 529, 530, 495, 499, 522, 526, 508, 562, 512, 519, 504, 523] payload_rows_per_dst=[5512, 5602, 5562, 5544, 5560, 5556, 5544, 5580] dup_rows_per_dst=[2600, 2628, 2698, 2592, 2538, 2618, 2630, 2772] combine_rows_per_src=[5512, 5602, 5562, 5544, 5560, 5556, 5544, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44576 payload_rows_remote=39098 dup_rows=20960 dup_rows_remote=18418 dedup_ratio=0.319824 router_tokens_per_expert=[511, 455, 499, 520, 536, 551, 508, 528, 502, 495, 534, 496, 512, 487, 513, 528, 538, 545, 534, 500, 510, 527, 551, 512, 508, 529, 507, 461, 515, 517, 475, 498, 489, 490, 513, 542, 483, 481, 507, 533, 520, 515, 530, 488, 509, 523, 562, 513, 509, 505, 500, 522, 526, 482, 513, 525, 518, 493, 480, 532, 548, 473, 540, 502] payload_rows_per_dst=[5536, 5526, 5676, 5478, 5554, 5668, 5586, 5552] dup_rows_per_dst=[2680, 2608, 2758, 2542, 2522, 2652, 2578, 2620] combine_rows_per_src=[5536, 5526, 5676, 5478, 5554, 5668, 5586, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44732 payload_rows_remote=39138 dup_rows=20804 dup_rows_remote=18214 dedup_ratio=0.317444 router_tokens_per_expert=[484, 526, 519, 536, 539, 499, 522, 496, 527, 514, 511, 482, 515, 478, 507, 504, 532, 506, 538, 492, 534, 527, 531, 546, 531, 520, 518, 524, 500, 483, 515, 499, 509, 504, 488, 525, 515, 505, 543, 529, 543, 511, 507, 463, 516, 519, 531, 502, 522, 488, 500, 500, 532, 516, 512, 492, 512, 511, 491, 506, 512, 488, 539, 482] payload_rows_per_dst=[5614, 5528, 5714, 5552, 5598, 5594, 5556, 5576] dup_rows_per_dst=[2628, 2548, 2698, 2628, 2638, 2590, 2568, 2506] combine_rows_per_src=[5614, 5528, 5714, 5552, 5598, 5594, 5556, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44618 payload_rows_remote=39012 dup_rows=20918 dup_rows_remote=18358 dedup_ratio=0.319183 router_tokens_per_expert=[508, 543, 484, 564, 496, 540, 527, 483, 503, 495, 519, 489, 512, 543, 535, 487, 501, 500, 510, 497, 524, 515, 499, 551, 507, 500, 547, 508, 547, 526, 502, 509, 489, 485, 524, 498, 509, 525, 509, 506, 493, 467, 525, 499, 488, 538, 514, 506, 536, 514, 545, 521, 513, 484, 547, 525, 482, 516, 524, 498, 513, 506, 501, 497] payload_rows_per_dst=[5636, 5606, 5540, 5634, 5458, 5538, 5664, 5542] dup_rows_per_dst=[2654, 2560, 2654, 2658, 2632, 2522, 2706, 2532] combine_rows_per_src=[5636, 5606, 5540, 5634, 5458, 5538, 5664, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159318016 dispatch_meta_bytes=229064 combine_payload_bytes=159318016 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159547080 ingress_bytes=209649664 link_bytes=209649664 token_bytes=318865096 weight_movement_bytes=50331648 total_nvlink_bytes=369196744 weight_to_token_ratio=0.157846 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595191706e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159047680 dispatch_meta_bytes=229456 combine_payload_bytes=159047680 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159277136 ingress_bytes=234545152 link_bytes=234545152 token_bytes=318324816 weight_movement_bytes=75497472 total_nvlink_bytes=393822288 weight_to_token_ratio=0.237171 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784617605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159571968 dispatch_meta_bytes=228880 combine_payload_bytes=159571968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159800848 ingress_bytes=184737792 link_bytes=184737792 token_bytes=319372816 weight_movement_bytes=25165824 total_nvlink_bytes=344538640 weight_to_token_ratio=0.078798 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405641144e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=38928 dup_rows=21014 dup_rows_remote=18288 dedup_ratio=0.320648 router_tokens_per_expert=[517, 484, 507, 528, 525, 536, 537, 502, 503, 478, 521, 492, 514, 516, 513, 510, 513, 544, 530, 507, 529, 514, 516, 479, 501, 508, 484, 513, 506, 508, 515, 521, 539, 561, 500, 478, 508, 520, 505, 510, 523, 498, 555, 537, 523, 535, 507, 492, 489, 527, 494, 507, 509, 492, 504, 507, 537, 518, 510, 500, 495, 503, 499, 515] payload_rows_per_dst=[5594, 5508, 5544, 5484, 5654, 5596, 5556, 5586] dup_rows_per_dst=[2726, 2586, 2720, 2628, 2588, 2744, 2454, 2568] combine_rows_per_src=[5594, 5508, 5544, 5484, 5654, 5596, 5556, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=229752 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159564152 ingress_bytes=184500224 link_bytes=184500224 token_bytes=318898552 weight_movement_bytes=25165824 total_nvlink_bytes=344064376 weight_to_token_ratio=0.078915 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403833526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160145408 dispatch_meta_bytes=230064 combine_payload_bytes=160145408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160375472 ingress_bytes=185311232 link_bytes=185311232 token_bytes=320520880 weight_movement_bytes=25165824 total_nvlink_bytes=345686704 weight_to_token_ratio=0.078515 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410004360e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160309248 dispatch_meta_bytes=229408 combine_payload_bytes=160309248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160538656 ingress_bytes=185475072 link_bytes=185475072 token_bytes=320847904 weight_movement_bytes=25165824 total_nvlink_bytes=346013728 weight_to_token_ratio=0.078435 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411250993e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159793152 dispatch_meta_bytes=229480 combine_payload_bytes=159793152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160022632 ingress_bytes=184958976 link_bytes=184958976 token_bytes=319815784 weight_movement_bytes=25165824 total_nvlink_bytes=344981608 weight_to_token_ratio=0.078688 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407324099e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159449088 dispatch_meta_bytes=228864 combine_payload_bytes=159449088 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159677952 ingress_bytes=260112384 link_bytes=260112384 token_bytes=319127040 weight_movement_bytes=100663296 total_nvlink_bytes=419790336 weight_to_token_ratio=0.315433 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.979154699e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38864 dup_rows=21114 dup_rows_remote=18404 dedup_ratio=0.322174 router_tokens_per_expert=[479, 476, 515, 534, 516, 502, 517, 479, 498, 520, 505, 518, 472, 548, 500, 519, 516, 498, 527, 506, 512, 494, 506, 569, 534, 501, 513, 535, 496, 496, 506, 504, 522, 505, 484, 525, 480, 504, 545, 532, 510, 487, 525, 532, 541, 456, 512, 527, 497, 513, 479, 510, 527, 553, 539, 516, 539, 537, 517, 484, 526, 492, 514, 527] payload_rows_per_dst=[5566, 5562, 5532, 5504, 5558, 5560, 5558, 5582] dup_rows_per_dst=[2470, 2598, 2724, 2666, 2636, 2620, 2710, 2690] combine_rows_per_src=[5566, 5562, 5532, 5504, 5558, 5560, 5558, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=38852 dup_rows=21070 dup_rows_remote=18420 dedup_ratio=0.321503 router_tokens_per_expert=[508, 506, 489, 521, 525, 526, 541, 535, 516, 530, 509, 506, 529, 541, 532, 455, 520, 512, 513, 521, 502, 548, 511, 505, 506, 527, 499, 538, 536, 513, 535, 501, 508, 496, 489, 496, 524, 462, 496, 508, 509, 526, 513, 528, 496, 507, 509, 490, 497, 482, 484, 509, 527, 510, 531, 504, 516, 487, 515, 524, 510, 542, 483, 534] payload_rows_per_dst=[5646, 5540, 5614, 5592, 5436, 5536, 5552, 5550] dup_rows_per_dst=[2656, 2696, 2650, 2718, 2522, 2620, 2536, 2672] combine_rows_per_src=[5646, 5540, 5614, 5592, 5436, 5536, 5552, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38886 dup_rows=21064 dup_rows_remote=18368 dedup_ratio=0.321411 router_tokens_per_expert=[469, 477, 503, 550, 532, 511, 506, 532, 522, 479, 499, 498, 541, 523, 529, 498, 530, 471, 531, 521, 493, 541, 523, 506, 524, 516, 500, 533, 523, 534, 519, 491, 495, 539, 504, 513, 541, 460, 494, 476, 521, 496, 559, 480, 517, 512, 522, 475, 491, 508, 530, 502, 500, 550, 522, 512, 501, 496, 508, 492, 519, 558, 522, 528] payload_rows_per_dst=[5586, 5544, 5524, 5630, 5482, 5506, 5614, 5586] dup_rows_per_dst=[2574, 2614, 2708, 2636, 2562, 2532, 2742, 2696] combine_rows_per_src=[5586, 5544, 5524, 5630, 5482, 5506, 5614, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44236 payload_rows_remote=38660 dup_rows=21300 dup_rows_remote=18596 dedup_ratio=0.325012 router_tokens_per_expert=[532, 493, 529, 498, 529, 494, 489, 495, 509, 532, 534, 480, 508, 514, 541, 520, 484, 534, 518, 505, 501, 509, 493, 497, 529, 526, 495, 511, 483, 494, 488, 514, 548, 502, 565, 510, 504, 492, 515, 504, 481, 501, 520, 520, 483, 511, 541, 527, 517, 507, 526, 536, 528, 537, 534, 481, 517, 512, 485, 518, 519, 523, 516, 510] payload_rows_per_dst=[5450, 5580, 5488, 5490, 5576, 5466, 5588, 5598] dup_rows_per_dst=[2668, 2696, 2594, 2590, 2704, 2702, 2744, 2602] combine_rows_per_src=[5450, 5580, 5488, 5490, 5576, 5466, 5588, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44766 payload_rows_remote=39122 dup_rows=20770 dup_rows_remote=18208 dedup_ratio=0.316925 router_tokens_per_expert=[500, 489, 506, 527, 500, 492, 538, 533, 512, 507, 483, 494, 501, 473, 512, 498, 541, 465, 500, 527, 510, 513, 519, 508, 491, 519, 518, 490, 510, 543, 491, 541, 514, 486, 497, 515, 526, 529, 552, 515, 484, 539, 498, 545, 530, 538, 535, 527, 497, 502, 546, 512, 500, 523, 537, 483, 513, 523, 516, 498, 508, 491, 529, 509] payload_rows_per_dst=[5618, 5522, 5512, 5644, 5630, 5620, 5594, 5626] dup_rows_per_dst=[2552, 2438, 2654, 2562, 2638, 2772, 2606, 2548] combine_rows_per_src=[5618, 5522, 5512, 5644, 5630, 5620, 5594, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159186944 dispatch_meta_bytes=229072 combine_payload_bytes=159186944 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159416016 ingress_bytes=209518592 link_bytes=209518592 token_bytes=318602960 weight_movement_bytes=50331648 total_nvlink_bytes=368934608 weight_to_token_ratio=0.157976 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594194400e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38766 dup_rows=21082 dup_rows_remote=18318 dedup_ratio=0.321686 router_tokens_per_expert=[512, 476, 519, 511, 478, 527, 494, 508, 533, 509, 477, 492, 533, 513, 531, 508, 498, 508, 524, 499, 513, 492, 503, 524, 536, 499, 517, 505, 499, 510, 497, 517, 524, 512, 510, 518, 542, 504, 489, 491, 566, 521, 500, 556, 532, 525, 524, 502, 496, 527, 490, 525, 531, 466, 537, 536, 518, 478, 509, 488, 534, 501, 515, 539] payload_rows_per_dst=[5528, 5498, 5566, 5516, 5594, 5688, 5520, 5544] dup_rows_per_dst=[2522, 2694, 2556, 2644, 2586, 2764, 2696, 2620] combine_rows_per_src=[5528, 5498, 5566, 5516, 5594, 5688, 5520, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44564 payload_rows_remote=38978 dup_rows=20972 dup_rows_remote=18326 dedup_ratio=0.320007 router_tokens_per_expert=[476, 480, 522, 547, 491, 518, 537, 531, 499, 532, 526, 499, 550, 467, 497, 546, 559, 518, 530, 514, 540, 524, 516, 534, 543, 527, 498, 486, 492, 483, 530, 547, 475, 501, 560, 530, 493, 481, 485, 494, 460, 503, 482, 530, 514, 517, 515, 494, 516, 479, 493, 520, 543, 551, 543, 532, 479, 508, 499, 530, 494, 508, 497, 483] payload_rows_per_dst=[5672, 5586, 5596, 5550, 5520, 5566, 5640, 5434] dup_rows_per_dst=[2532, 2646, 2874, 2662, 2518, 2464, 2714, 2562] combine_rows_per_src=[5672, 5586, 5596, 5550, 5520, 5566, 5640, 5434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159137792 dispatch_meta_bytes=229088 combine_payload_bytes=159137792 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159366880 ingress_bytes=209469440 link_bytes=209469440 token_bytes=318504672 weight_movement_bytes=50331648 total_nvlink_bytes=368836320 weight_to_token_ratio=0.158025 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593820410e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=229016 combine_payload_bytes=159277056 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159506072 ingress_bytes=184442880 link_bytes=184442880 token_bytes=318783128 weight_movement_bytes=25165824 total_nvlink_bytes=343948952 weight_to_token_ratio=0.078943 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403397205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158351360 dispatch_meta_bytes=229024 combine_payload_bytes=158351360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158580384 ingress_bytes=183517184 link_bytes=183517184 token_bytes=316931744 weight_movement_bytes=25165824 total_nvlink_bytes=342097568 weight_to_token_ratio=0.079405 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396353728e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160243712 dispatch_meta_bytes=229320 combine_payload_bytes=160243712 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160473032 ingress_bytes=235741184 link_bytes=235741184 token_bytes=320716744 weight_movement_bytes=75497472 total_nvlink_bytes=396214216 weight_to_token_ratio=0.235402 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.793718026e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44402 payload_rows_remote=38722 dup_rows=21134 dup_rows_remote=18318 dedup_ratio=0.322479 router_tokens_per_expert=[487, 508, 512, 525, 533, 512, 504, 505, 522, 509, 542, 524, 512, 514, 503, 530, 524, 518, 510, 515, 493, 509, 535, 485, 529, 511, 502, 513, 501, 528, 527, 500, 531, 508, 523, 501, 490, 521, 548, 513, 494, 500, 517, 498, 521, 535, 509, 491, 492, 492, 499, 509, 516, 495, 519, 491, 514, 528, 525, 487, 518, 499, 539, 503] payload_rows_per_dst=[5680, 5588, 5470, 5520, 5680, 5508, 5398, 5558] dup_rows_per_dst=[2816, 2734, 2474, 2612, 2894, 2612, 2324, 2668] combine_rows_per_src=[5680, 5588, 5470, 5520, 5680, 5508, 5398, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158785536 dispatch_meta_bytes=228336 combine_payload_bytes=158785536 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159013872 ingress_bytes=209117184 link_bytes=209117184 token_bytes=317799408 weight_movement_bytes=50331648 total_nvlink_bytes=368131056 weight_to_token_ratio=0.158376 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591140149e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159653888 dispatch_meta_bytes=229216 combine_payload_bytes=159653888 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159883104 ingress_bytes=209985536 link_bytes=209985536 token_bytes=319536992 weight_movement_bytes=50331648 total_nvlink_bytes=369868640 weight_to_token_ratio=0.157514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597747304e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38948 dup_rows=21028 dup_rows_remote=18410 dedup_ratio=0.320862 router_tokens_per_expert=[529, 516, 520, 495, 512, 527, 525, 478, 496, 482, 508, 484, 470, 536, 510, 495, 491, 465, 510, 500, 535, 512, 554, 474, 549, 504, 551, 532, 536, 572, 521, 514, 514, 520, 510, 521, 504, 509, 538, 512, 506, 496, 522, 507, 536, 486, 484, 536, 570, 516, 510, 508, 482, 511, 530, 462, 479, 520, 504, 544, 524, 489, 513, 502] payload_rows_per_dst=[5532, 5506, 5480, 5734, 5570, 5552, 5560, 5574] dup_rows_per_dst=[2672, 2456, 2602, 2824, 2686, 2594, 2618, 2576] combine_rows_per_src=[5532, 5506, 5480, 5734, 5570, 5552, 5560, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158605312 dispatch_meta_bytes=228160 combine_payload_bytes=158605312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158833472 ingress_bytes=183771136 link_bytes=183771136 token_bytes=317438784 weight_movement_bytes=25165824 total_nvlink_bytes=342604608 weight_to_token_ratio=0.079278 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398286009e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44276 payload_rows_remote=38804 dup_rows=21260 dup_rows_remote=18564 dedup_ratio=0.324402 router_tokens_per_expert=[532, 535, 532, 494, 512, 541, 464, 516, 502, 509, 481, 496, 525, 526, 510, 507, 520, 493, 516, 505, 518, 489, 521, 522, 515, 545, 530, 525, 563, 511, 492, 549, 508, 525, 512, 519, 516, 485, 511, 526, 482, 518, 510, 556, 520, 509, 489, 469, 509, 547, 531, 465, 533, 510, 499, 513, 486, 502, 502, 489, 529, 505, 493, 504] payload_rows_per_dst=[5560, 5434, 5472, 5662, 5580, 5532, 5552, 5484] dup_rows_per_dst=[2692, 2678, 2696, 2798, 2624, 2574, 2662, 2536] combine_rows_per_src=[5560, 5434, 5472, 5662, 5580, 5532, 5552, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38752 dup_rows=21108 dup_rows_remote=18238 dedup_ratio=0.322083 router_tokens_per_expert=[512, 513, 572, 513, 529, 524, 514, 516, 487, 522, 510, 508, 537, 529, 487, 501, 514, 540, 526, 508, 525, 484, 536, 543, 485, 500, 513, 505, 516, 511, 499, 547, 524, 485, 496, 504, 484, 508, 515, 505, 537, 505, 484, 532, 509, 519, 501, 550, 499, 522, 520, 488, 468, 521, 498, 495, 540, 522, 490, 496, 486, 528, 495, 516] payload_rows_per_dst=[5588, 5534, 5652, 5418, 5478, 5612, 5470, 5676] dup_rows_per_dst=[2750, 2628, 2700, 2360, 2494, 2754, 2552, 2870] combine_rows_per_src=[5588, 5534, 5652, 5418, 5478, 5612, 5470, 5676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159531008 dispatch_meta_bytes=229432 combine_payload_bytes=159531008 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159760440 ingress_bytes=184696832 link_bytes=184696832 token_bytes=319291448 weight_movement_bytes=25165824 total_nvlink_bytes=344457272 weight_to_token_ratio=0.078818 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405329486e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38946 dup_rows=21082 dup_rows_remote=18484 dedup_ratio=0.321686 router_tokens_per_expert=[487, 512, 512, 510, 513, 509, 511, 520, 535, 519, 521, 565, 520, 494, 504, 534, 487, 519, 510, 526, 520, 503, 467, 497, 520, 514, 498, 489, 527, 523, 501, 525, 493, 494, 495, 538, 487, 494, 524, 528, 530, 524, 533, 507, 500, 491, 465, 557, 520, 473, 549, 518, 527, 526, 536, 522, 493, 515, 544, 506, 496, 481, 492, 518] payload_rows_per_dst=[5570, 5616, 5550, 5524, 5508, 5558, 5594, 5534] dup_rows_per_dst=[2578, 2768, 2508, 2670, 2598, 2656, 2748, 2556] combine_rows_per_src=[5570, 5616, 5550, 5524, 5508, 5558, 5594, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44652 payload_rows_remote=39074 dup_rows=20884 dup_rows_remote=18258 dedup_ratio=0.318665 router_tokens_per_expert=[511, 499, 515, 528, 538, 507, 518, 504, 515, 539, 526, 456, 539, 522, 536, 498, 500, 490, 502, 477, 535, 490, 538, 500, 505, 482, 511, 530, 494, 551, 539, 490, 537, 544, 509, 505, 494, 467, 508, 522, 519, 506, 539, 507, 570, 533, 501, 508, 522, 525, 472, 497, 491, 476, 542, 515, 504, 488, 518, 497, 525, 533, 516, 493] payload_rows_per_dst=[5532, 5616, 5554, 5578, 5630, 5646, 5512, 5584] dup_rows_per_dst=[2708, 2646, 2510, 2626, 2542, 2720, 2568, 2564] combine_rows_per_src=[5532, 5616, 5554, 5578, 5630, 5646, 5512, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44542 payload_rows_remote=39004 dup_rows=20994 dup_rows_remote=18470 dedup_ratio=0.320343 router_tokens_per_expert=[485, 484, 507, 538, 545, 505, 486, 503, 523, 458, 519, 488, 493, 500, 542, 508, 477, 505, 463, 498, 499, 502, 503, 534, 503, 551, 544, 521, 520, 492, 505, 565, 513, 534, 531, 520, 522, 518, 505, 538, 526, 483, 492, 491, 503, 503, 543, 508, 556, 489, 532, 495, 504, 492, 517, 548, 481, 523, 498, 555, 508, 524, 531, 519] payload_rows_per_dst=[5504, 5538, 5536, 5686, 5610, 5528, 5570, 5570] dup_rows_per_dst=[2602, 2524, 2426, 2716, 2752, 2570, 2696, 2708] combine_rows_per_src=[5504, 5538, 5536, 5686, 5610, 5528, 5570, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44628 payload_rows_remote=39012 dup_rows=20908 dup_rows_remote=18280 dedup_ratio=0.319031 router_tokens_per_expert=[474, 532, 493, 479, 498, 487, 507, 504, 515, 563, 510, 536, 510, 525, 455, 477, 542, 541, 470, 508, 502, 511, 544, 532, 534, 510, 540, 496, 493, 524, 514, 496, 499, 530, 490, 504, 532, 525, 538, 534, 530, 523, 532, 528, 511, 493, 485, 520, 510, 516, 516, 527, 531, 529, 476, 498, 492, 494, 489, 516, 490, 530, 536, 522] payload_rows_per_dst=[5460, 5594, 5602, 5632, 5610, 5616, 5586, 5528] dup_rows_per_dst=[2488, 2588, 2698, 2582, 2694, 2628, 2620, 2610] combine_rows_per_src=[5460, 5594, 5602, 5632, 5610, 5616, 5586, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158728192 dispatch_meta_bytes=227960 combine_payload_bytes=158728192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158956152 ingress_bytes=183894016 link_bytes=183894016 token_bytes=317684344 weight_movement_bytes=25165824 total_nvlink_bytes=342850168 weight_to_token_ratio=0.079216 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399220984e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=229472 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159170656 ingress_bytes=184107008 link_bytes=184107008 token_bytes=318111840 weight_movement_bytes=25165824 total_nvlink_bytes=343277664 weight_to_token_ratio=0.079110 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400841607e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229720 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159752536 ingress_bytes=209854464 link_bytes=209854464 token_bytes=319275352 weight_movement_bytes=50331648 total_nvlink_bytes=369607000 weight_to_token_ratio=0.157643 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596749998e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=160047104 dispatch_meta_bytes=229328 combine_payload_bytes=160047104 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160276432 ingress_bytes=235544576 link_bytes=235544576 token_bytes=320323536 weight_movement_bytes=75497472 total_nvlink_bytes=395821008 weight_to_token_ratio=0.235691 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792222067e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44494 payload_rows_remote=38916 dup_rows=21042 dup_rows_remote=18320 dedup_ratio=0.321075 router_tokens_per_expert=[498, 566, 493, 515, 503, 527, 527, 520, 550, 490, 531, 532, 501, 506, 510, 499, 490, 547, 498, 474, 494, 524, 514, 506, 497, 491, 516, 573, 514, 519, 499, 482, 534, 550, 524, 503, 464, 515, 475, 515, 533, 518, 490, 501, 500, 524, 536, 482, 524, 511, 519, 508, 464, 564, 515, 510, 500, 511, 501, 513, 510, 531, 504, 513] payload_rows_per_dst=[5578, 5614, 5706, 5426, 5594, 5502, 5620, 5454] dup_rows_per_dst=[2722, 2796, 2844, 2300, 2566, 2494, 2608, 2712] combine_rows_per_src=[5578, 5614, 5706, 5426, 5594, 5502, 5620, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159760384 dispatch_meta_bytes=229896 combine_payload_bytes=159760384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159990280 ingress_bytes=184926208 link_bytes=184926208 token_bytes=319750664 weight_movement_bytes=25165824 total_nvlink_bytes=344916488 weight_to_token_ratio=0.078705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407074772e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159793152 dispatch_meta_bytes=229168 combine_payload_bytes=159793152 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160022320 ingress_bytes=210124800 link_bytes=210124800 token_bytes=319815472 weight_movement_bytes=50331648 total_nvlink_bytes=370147120 weight_to_token_ratio=0.157377 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598806942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38884 dup_rows=21100 dup_rows_remote=18396 dedup_ratio=0.321960 router_tokens_per_expert=[469, 500, 514, 560, 483, 511, 497, 534, 523, 517, 491, 532, 518, 503, 508, 513, 525, 506, 518, 497, 480, 518, 508, 550, 499, 541, 523, 561, 510, 546, 475, 495, 470, 531, 500, 507, 488, 501, 480, 521, 519, 518, 459, 502, 501, 522, 510, 550, 513, 523, 528, 492, 489, 507, 495, 581, 509, 519, 515, 513, 487, 519, 566, 508] payload_rows_per_dst=[5514, 5624, 5534, 5578, 5428, 5582, 5552, 5624] dup_rows_per_dst=[2622, 2586, 2670, 2722, 2568, 2580, 2704, 2648] combine_rows_per_src=[5514, 5624, 5534, 5578, 5428, 5582, 5552, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159399936 dispatch_meta_bytes=228944 combine_payload_bytes=159399936 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159628880 ingress_bytes=234897408 link_bytes=234897408 token_bytes=319028816 weight_movement_bytes=75497472 total_nvlink_bytes=394526288 weight_to_token_ratio=0.236648 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787297866e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=229120 combine_payload_bytes=159268864 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159497984 ingress_bytes=234766336 link_bytes=234766336 token_bytes=318766848 weight_movement_bytes=75497472 total_nvlink_bytes=394264320 weight_to_token_ratio=0.236842 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786300559e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44320 payload_rows_remote=38654 dup_rows=21216 dup_rows_remote=18340 dedup_ratio=0.323730 router_tokens_per_expert=[489, 491, 493, 490, 495, 495, 533, 542, 500, 538, 516, 503, 505, 507, 529, 523, 514, 504, 486, 497, 528, 528, 501, 492, 530, 514, 480, 524, 548, 494, 535, 525, 501, 483, 559, 526, 531, 502, 535, 528, 500, 513, 507, 536, 553, 531, 545, 499, 483, 500, 480, 490, 521, 525, 529, 501, 492, 518, 495, 509, 535, 476, 494, 522] payload_rows_per_dst=[5434, 5512, 5470, 5660, 5396, 5638, 5544, 5666] dup_rows_per_dst=[2622, 2730, 2550, 2720, 2474, 2730, 2514, 2876] combine_rows_per_src=[5434, 5512, 5470, 5660, 5396, 5638, 5544, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44614 payload_rows_remote=39016 dup_rows=20922 dup_rows_remote=18332 dedup_ratio=0.319244 router_tokens_per_expert=[511, 498, 492, 519, 495, 546, 545, 552, 498, 514, 551, 526, 511, 514, 542, 508, 480, 492, 507, 503, 547, 503, 544, 514, 477, 491, 502, 526, 470, 506, 483, 512, 498, 509, 526, 524, 515, 510, 500, 512, 510, 559, 452, 564, 509, 526, 497, 507, 498, 545, 513, 518, 490, 521, 519, 527, 522, 538, 494, 499, 505, 497, 492, 493] payload_rows_per_dst=[5714, 5662, 5538, 5472, 5598, 5492, 5592, 5546] dup_rows_per_dst=[2602, 2666, 2642, 2462, 2590, 2756, 2670, 2534] combine_rows_per_src=[5714, 5662, 5538, 5472, 5598, 5492, 5592, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44604 payload_rows_remote=39028 dup_rows=20932 dup_rows_remote=18214 dedup_ratio=0.319397 router_tokens_per_expert=[491, 494, 505, 498, 519, 502, 512, 488, 498, 458, 511, 558, 486, 522, 541, 516, 513, 505, 480, 554, 521, 516, 477, 581, 517, 517, 485, 548, 523, 516, 532, 521, 540, 479, 524, 533, 498, 502, 504, 510, 538, 494, 512, 524, 526, 491, 448, 517, 537, 530, 511, 517, 478, 526, 518, 507, 493, 515, 499, 521, 510, 497, 536, 528] payload_rows_per_dst=[5480, 5530, 5576, 5674, 5602, 5594, 5588, 5560] dup_rows_per_dst=[2538, 2650, 2718, 2644, 2578, 2506, 2660, 2638] combine_rows_per_src=[5480, 5530, 5576, 5674, 5602, 5594, 5588, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=38996 dup_rows=21014 dup_rows_remote=18458 dedup_ratio=0.320648 router_tokens_per_expert=[550, 486, 489, 528, 506, 510, 521, 519, 536, 499, 501, 520, 502, 499, 509, 482, 499, 516, 529, 510, 495, 520, 505, 540, 504, 524, 480, 500, 518, 516, 479, 520, 476, 568, 537, 527, 545, 529, 483, 522, 501, 512, 535, 514, 522, 526, 516, 528, 480, 528, 506, 539, 534, 493, 497, 514, 510, 483, 504, 499, 497, 519, 527, 485] payload_rows_per_dst=[5536, 5568, 5602, 5526, 5606, 5624, 5536, 5524] dup_rows_per_dst=[2682, 2528, 2626, 2556, 2768, 2684, 2646, 2524] combine_rows_per_src=[5536, 5568, 5602, 5526, 5606, 5624, 5536, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158326784 dispatch_meta_bytes=227976 combine_payload_bytes=158326784 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158554760 ingress_bytes=183492608 link_bytes=183492608 token_bytes=316881544 weight_movement_bytes=25165824 total_nvlink_bytes=342047368 weight_to_token_ratio=0.079417 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396166733e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44442 payload_rows_remote=38820 dup_rows=21094 dup_rows_remote=18436 dedup_ratio=0.321869 router_tokens_per_expert=[507, 472, 518, 530, 526, 493, 521, 505, 517, 561, 556, 509, 488, 517, 492, 500, 515, 513, 514, 521, 528, 480, 503, 535, 469, 536, 526, 487, 512, 532, 493, 495, 511, 518, 530, 538, 457, 507, 522, 512, 534, 499, 500, 497, 513, 504, 518, 526, 509, 518, 501, 499, 514, 486, 533, 504, 529, 523, 519, 539, 515, 513, 497, 512] payload_rows_per_dst=[5520, 5622, 5494, 5548, 5524, 5552, 5556, 5626] dup_rows_per_dst=[2624, 2658, 2724, 2552, 2666, 2630, 2572, 2668] combine_rows_per_src=[5520, 5622, 5494, 5548, 5524, 5552, 5556, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44368 payload_rows_remote=38920 dup_rows=21168 dup_rows_remote=18536 dedup_ratio=0.322998 router_tokens_per_expert=[537, 494, 570, 495, 534, 512, 546, 532, 526, 530, 485, 495, 514, 516, 481, 476, 494, 541, 467, 506, 524, 534, 513, 507, 513, 491, 533, 465, 538, 483, 534, 530, 536, 519, 524, 525, 509, 549, 532, 510, 513, 492, 500, 488, 515, 517, 535, 480, 488, 532, 507, 499, 515, 529, 512, 506, 505, 480, 482, 514, 507, 485, 533, 514] payload_rows_per_dst=[5610, 5516, 5564, 5494, 5694, 5448, 5550, 5492] dup_rows_per_dst=[2830, 2530, 2608, 2680, 2714, 2632, 2626, 2548] combine_rows_per_src=[5610, 5516, 5564, 5494, 5694, 5448, 5550, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159809536 dispatch_meta_bytes=229392 combine_payload_bytes=159809536 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160038928 ingress_bytes=210141184 link_bytes=210141184 token_bytes=319848464 weight_movement_bytes=50331648 total_nvlink_bytes=370180112 weight_to_token_ratio=0.157361 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598931606e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159858688 dispatch_meta_bytes=228968 combine_payload_bytes=159858688 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160087656 ingress_bytes=210190336 link_bytes=210190336 token_bytes=319946344 weight_movement_bytes=50331648 total_nvlink_bytes=370277992 weight_to_token_ratio=0.157313 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599305595e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159727616 dispatch_meta_bytes=229816 combine_payload_bytes=159727616 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159957432 ingress_bytes=184893440 link_bytes=184893440 token_bytes=319685048 weight_movement_bytes=25165824 total_nvlink_bytes=344850872 weight_to_token_ratio=0.078721 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406825446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44618 payload_rows_remote=38996 dup_rows=20918 dup_rows_remote=18332 dedup_ratio=0.319183 router_tokens_per_expert=[501, 517, 522, 504, 497, 522, 520, 506, 494, 518, 516, 503, 508, 507, 548, 497, 499, 527, 501, 496, 548, 490, 535, 512, 515, 488, 521, 512, 514, 532, 538, 504, 502, 518, 527, 489, 509, 531, 527, 503, 505, 538, 516, 495, 508, 480, 515, 543, 489, 523, 511, 510, 529, 538, 497, 522, 515, 486, 536, 508, 495, 517, 486, 488] payload_rows_per_dst=[5622, 5520, 5536, 5672, 5628, 5538, 5566, 5536] dup_rows_per_dst=[2586, 2634, 2684, 2576, 2702, 2550, 2660, 2526] combine_rows_per_src=[5622, 5520, 5536, 5672, 5628, 5538, 5566, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38822 dup_rows=21064 dup_rows_remote=18418 dedup_ratio=0.321411 router_tokens_per_expert=[493, 493, 501, 531, 559, 509, 507, 528, 538, 513, 514, 498, 477, 528, 485, 542, 534, 517, 535, 495, 528, 519, 532, 541, 487, 523, 499, 501, 524, 482, 512, 496, 514, 541, 505, 502, 526, 490, 473, 502, 486, 533, 534, 519, 504, 508, 486, 501, 547, 517, 521, 528, 537, 539, 487, 472, 504, 507, 501, 494, 528, 509, 506, 506] payload_rows_per_dst=[5610, 5496, 5638, 5534, 5568, 5512, 5650, 5464] dup_rows_per_dst=[2632, 2694, 2764, 2514, 2538, 2630, 2646, 2646] combine_rows_per_src=[5610, 5496, 5638, 5534, 5568, 5512, 5650, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=229024 combine_payload_bytes=159006720 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159235744 ingress_bytes=209338368 link_bytes=209338368 token_bytes=318242464 weight_movement_bytes=50331648 total_nvlink_bytes=368574112 weight_to_token_ratio=0.158155 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592823103e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=229824 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159646144 ingress_bytes=209747968 link_bytes=209747968 token_bytes=319062464 weight_movement_bytes=50331648 total_nvlink_bytes=369394112 weight_to_token_ratio=0.157749 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595939686e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44624 payload_rows_remote=38970 dup_rows=20912 dup_rows_remote=18276 dedup_ratio=0.319092 router_tokens_per_expert=[523, 510, 535, 511, 482, 503, 493, 518, 515, 486, 530, 528, 504, 508, 475, 526, 518, 515, 520, 510, 512, 537, 495, 503, 556, 518, 522, 523, 498, 482, 545, 506, 471, 520, 497, 550, 497, 522, 486, 502, 479, 530, 517, 478, 534, 500, 512, 500, 509, 494, 554, 516, 547, 518, 514, 472, 499, 514, 543, 546, 497, 495, 534, 514] payload_rows_per_dst=[5498, 5498, 5606, 5668, 5508, 5498, 5694, 5654] dup_rows_per_dst=[2376, 2578, 2682, 2908, 2364, 2596, 2772, 2636] combine_rows_per_src=[5498, 5498, 5606, 5668, 5508, 5498, 5694, 5654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159727616 dispatch_meta_bytes=229312 combine_payload_bytes=159727616 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159956928 ingress_bytes=184893440 link_bytes=184893440 token_bytes=319684544 weight_movement_bytes=25165824 total_nvlink_bytes=344850368 weight_to_token_ratio=0.078721 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406825446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159014912 dispatch_meta_bytes=228960 combine_payload_bytes=159014912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159243872 ingress_bytes=184180736 link_bytes=184180736 token_bytes=318258784 weight_movement_bytes=25165824 total_nvlink_bytes=343424608 weight_to_token_ratio=0.079073 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401402592e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38950 dup_rows=21120 dup_rows_remote=18524 dedup_ratio=0.322266 router_tokens_per_expert=[536, 521, 515, 517, 530, 501, 507, 539, 523, 474, 476, 520, 479, 504, 516, 526, 476, 476, 509, 530, 517, 506, 514, 506, 483, 534, 499, 553, 509, 499, 519, 536, 507, 509, 528, 525, 479, 508, 490, 485, 548, 523, 485, 531, 532, 518, 560, 521, 484, 463, 567, 520, 492, 512, 503, 533, 519, 479, 468, 506, 521, 530, 533, 539] payload_rows_per_dst=[5598, 5488, 5484, 5628, 5466, 5662, 5530, 5560] dup_rows_per_dst=[2734, 2548, 2584, 2636, 2596, 2774, 2618, 2630] combine_rows_per_src=[5598, 5488, 5484, 5628, 5466, 5662, 5530, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44640 payload_rows_remote=39084 dup_rows=20896 dup_rows_remote=18382 dedup_ratio=0.318848 router_tokens_per_expert=[451, 484, 557, 506, 518, 513, 522, 531, 508, 521, 510, 546, 532, 473, 521, 508, 489, 517, 497, 500, 514, 505, 494, 519, 518, 534, 494, 528, 516, 507, 523, 473, 496, 525, 535, 500, 513, 534, 514, 501, 516, 481, 524, 515, 573, 528, 468, 557, 520, 492, 500, 478, 542, 501, 508, 494, 522, 493, 498, 514, 523, 528, 532, 514] payload_rows_per_dst=[5554, 5622, 5556, 5582, 5618, 5654, 5490, 5564] dup_rows_per_dst=[2610, 2616, 2514, 2604, 2618, 2670, 2580, 2684] combine_rows_per_src=[5554, 5622, 5556, 5582, 5618, 5654, 5490, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44386 payload_rows_remote=38924 dup_rows=21150 dup_rows_remote=18528 dedup_ratio=0.322723 router_tokens_per_expert=[499, 512, 539, 525, 515, 514, 536, 511, 518, 517, 522, 515, 523, 517, 525, 533, 544, 500, 515, 500, 501, 515, 524, 521, 462, 533, 512, 507, 485, 504, 521, 518, 525, 515, 541, 476, 491, 521, 510, 537, 495, 567, 516, 452, 523, 507, 539, 521, 481, 504, 544, 486, 507, 507, 501, 510, 499, 506, 476, 511, 504, 521, 509, 483] payload_rows_per_dst=[5562, 5562, 5594, 5462, 5574, 5570, 5520, 5542] dup_rows_per_dst=[2740, 2778, 2646, 2622, 2658, 2670, 2560, 2476] combine_rows_per_src=[5562, 5562, 5594, 5462, 5574, 5570, 5520, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159621120 dispatch_meta_bytes=228984 combine_payload_bytes=159621120 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159850104 ingress_bytes=209952768 link_bytes=209952768 token_bytes=319471224 weight_movement_bytes=50331648 total_nvlink_bytes=369802872 weight_to_token_ratio=0.157547 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597497977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44580 payload_rows_remote=39022 dup_rows=20956 dup_rows_remote=18272 dedup_ratio=0.319763 router_tokens_per_expert=[502, 554, 506, 531, 476, 478, 520, 546, 515, 517, 516, 522, 513, 498, 522, 507, 498, 459, 500, 507, 529, 525, 486, 494, 508, 499, 534, 503, 520, 532, 491, 494, 511, 487, 516, 490, 538, 543, 530, 529, 505, 510, 533, 504, 550, 490, 484, 545, 497, 510, 516, 491, 554, 489, 505, 511, 506, 549, 517, 523, 535, 518, 484, 496] payload_rows_per_dst=[5584, 5624, 5468, 5638, 5574, 5558, 5568, 5566] dup_rows_per_dst=[2642, 2596, 2528, 2524, 2714, 2684, 2578, 2690] combine_rows_per_src=[5584, 5624, 5468, 5638, 5574, 5558, 5568, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38886 dup_rows=21098 dup_rows_remote=18418 dedup_ratio=0.321930 router_tokens_per_expert=[487, 510, 501, 516, 517, 476, 497, 526, 488, 491, 521, 537, 513, 511, 527, 528, 585, 516, 515, 513, 543, 523, 525, 538, 507, 501, 518, 505, 524, 504, 507, 493, 540, 537, 522, 512, 520, 532, 507, 480, 514, 483, 492, 521, 503, 511, 473, 486, 528, 513, 520, 497, 470, 528, 509, 493, 505, 532, 531, 516, 501, 486, 548, 495] payload_rows_per_dst=[5504, 5552, 5694, 5506, 5622, 5498, 5498, 5564] dup_rows_per_dst=[2556, 2680, 2822, 2612, 2678, 2468, 2618, 2664] combine_rows_per_src=[5504, 5552, 5694, 5506, 5622, 5498, 5498, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159539200 dispatch_meta_bytes=229896 combine_payload_bytes=159539200 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159769096 ingress_bytes=209870848 link_bytes=209870848 token_bytes=319308296 weight_movement_bytes=50331648 total_nvlink_bytes=369639944 weight_to_token_ratio=0.157627 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596874661e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=160088064 dispatch_meta_bytes=229864 combine_payload_bytes=160088064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160317928 ingress_bytes=185253888 link_bytes=185253888 token_bytes=320405992 weight_movement_bytes=25165824 total_nvlink_bytes=345571816 weight_to_token_ratio=0.078544 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409568039e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=229808 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159662512 ingress_bytes=184598528 link_bytes=184598528 token_bytes=319095216 weight_movement_bytes=25165824 total_nvlink_bytes=344261040 weight_to_token_ratio=0.078866 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404581506e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44382 payload_rows_remote=38786 dup_rows=21154 dup_rows_remote=18414 dedup_ratio=0.322784 router_tokens_per_expert=[491, 513, 529, 501, 513, 512, 525, 541, 516, 499, 501, 503, 539, 530, 500, 513, 500, 491, 537, 460, 442, 506, 512, 530, 527, 488, 509, 481, 537, 490, 509, 508, 510, 517, 483, 523, 536, 501, 554, 545, 537, 518, 542, 479, 502, 478, 507, 535, 528, 566, 477, 488, 542, 516, 483, 507, 511, 528, 487, 537, 533, 516, 499, 530] payload_rows_per_dst=[5596, 5594, 5470, 5506, 5582, 5490, 5602, 5542] dup_rows_per_dst=[2740, 2610, 2486, 2592, 2858, 2602, 2612, 2654] combine_rows_per_src=[5596, 5594, 5470, 5506, 5582, 5490, 5602, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159834112 dispatch_meta_bytes=229176 combine_payload_bytes=159834112 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160063288 ingress_bytes=235331584 link_bytes=235331584 token_bytes=319897400 weight_movement_bytes=75497472 total_nvlink_bytes=395394872 weight_to_token_ratio=0.236005 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790601444e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44396 payload_rows_remote=38888 dup_rows=21140 dup_rows_remote=18564 dedup_ratio=0.322571 router_tokens_per_expert=[511, 503, 491, 538, 530, 514, 491, 524, 497, 505, 548, 508, 520, 537, 532, 511, 507, 502, 505, 507, 505, 499, 497, 484, 508, 497, 535, 533, 518, 526, 539, 512, 514, 518, 539, 520, 494, 518, 550, 489, 504, 520, 496, 512, 510, 513, 465, 526, 486, 499, 528, 511, 523, 486, 511, 498, 511, 518, 509, 500, 499, 497, 551, 519] payload_rows_per_dst=[5494, 5516, 5502, 5578, 5634, 5602, 5508, 5562] dup_rows_per_dst=[2710, 2800, 2510, 2758, 2650, 2490, 2576, 2646] combine_rows_per_src=[5494, 5516, 5502, 5578, 5634, 5602, 5508, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=229216 combine_payload_bytes=159277056 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159506272 ingress_bytes=209608704 link_bytes=209608704 token_bytes=318783328 weight_movement_bytes=50331648 total_nvlink_bytes=369114976 weight_to_token_ratio=0.157887 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594880048e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44346 payload_rows_remote=38686 dup_rows=21190 dup_rows_remote=18496 dedup_ratio=0.323334 router_tokens_per_expert=[510, 504, 503, 530, 514, 522, 513, 489, 561, 484, 482, 520, 484, 527, 532, 487, 492, 478, 477, 519, 525, 492, 512, 498, 508, 526, 480, 524, 502, 525, 505, 531, 505, 496, 503, 538, 492, 544, 517, 519, 520, 508, 530, 487, 502, 526, 549, 531, 511, 533, 475, 512, 499, 539, 529, 508, 493, 551, 502, 499, 585, 516, 468, 525] payload_rows_per_dst=[5596, 5506, 5458, 5534, 5544, 5516, 5532, 5660] dup_rows_per_dst=[2574, 2648, 2514, 2668, 2688, 2790, 2614, 2694] combine_rows_per_src=[5596, 5506, 5458, 5534, 5544, 5516, 5532, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158867456 dispatch_meta_bytes=228800 combine_payload_bytes=158867456 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159096256 ingress_bytes=209199104 link_bytes=209199104 token_bytes=317963712 weight_movement_bytes=50331648 total_nvlink_bytes=368295360 weight_to_token_ratio=0.158294 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591763465e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44380 payload_rows_remote=38770 dup_rows=21156 dup_rows_remote=18540 dedup_ratio=0.322815 router_tokens_per_expert=[484, 511, 489, 515, 492, 545, 494, 539, 539, 494, 501, 515, 539, 500, 535, 488, 519, 522, 499, 561, 485, 524, 464, 491, 530, 514, 490, 481, 520, 537, 484, 506, 505, 514, 524, 519, 513, 497, 520, 521, 499, 533, 521, 552, 524, 505, 550, 483, 484, 522, 528, 550, 518, 484, 479, 511, 542, 519, 496, 460, 519, 540, 547, 482] payload_rows_per_dst=[5538, 5608, 5524, 5480, 5610, 5580, 5510, 5530] dup_rows_per_dst=[2600, 2614, 2606, 2644, 2616, 2754, 2642, 2680] combine_rows_per_src=[5538, 5608, 5524, 5480, 5610, 5580, 5510, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=229808 combine_payload_bytes=159285248 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159515056 ingress_bytes=234782720 link_bytes=234782720 token_bytes=318800304 weight_movement_bytes=75497472 total_nvlink_bytes=394297776 weight_to_token_ratio=0.236817 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786425223e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44318 payload_rows_remote=38700 dup_rows=21218 dup_rows_remote=18510 dedup_ratio=0.323761 router_tokens_per_expert=[480, 497, 479, 545, 526, 514, 508, 466, 557, 526, 503, 573, 548, 543, 514, 513, 539, 537, 518, 473, 500, 553, 529, 514, 535, 509, 488, 545, 526, 477, 474, 480, 504, 496, 492, 487, 453, 490, 505, 483, 528, 497, 500, 555, 543, 511, 487, 539, 537, 562, 493, 505, 485, 540, 513, 510, 508, 498, 523, 498, 517, 506, 523, 491] payload_rows_per_dst=[5522, 5590, 5618, 5492, 5372, 5620, 5550, 5554] dup_rows_per_dst=[2508, 2964, 2708, 2576, 2448, 2700, 2740, 2574] combine_rows_per_src=[5522, 5590, 5618, 5492, 5372, 5620, 5550, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158457856 dispatch_meta_bytes=228728 combine_payload_bytes=158457856 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158686584 ingress_bytes=208789504 link_bytes=208789504 token_bytes=317144440 weight_movement_bytes=50331648 total_nvlink_bytes=367476088 weight_to_token_ratio=0.158703 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588646882e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44526 payload_rows_remote=38928 dup_rows=21010 dup_rows_remote=18292 dedup_ratio=0.320587 router_tokens_per_expert=[490, 536, 478, 544, 522, 525, 537, 511, 515, 520, 469, 513, 497, 486, 489, 512, 491, 522, 512, 510, 513, 558, 546, 519, 517, 520, 534, 521, 501, 511, 531, 523, 535, 526, 496, 473, 543, 543, 521, 502, 479, 496, 474, 510, 503, 496, 519, 516, 488, 514, 529, 545, 527, 461, 531, 527, 496, 449, 545, 499, 541, 532, 499, 480] payload_rows_per_dst=[5576, 5572, 5628, 5598, 5574, 5450, 5628, 5500] dup_rows_per_dst=[2710, 2430, 2714, 2718, 2704, 2536, 2616, 2582] combine_rows_per_src=[5576, 5572, 5628, 5598, 5574, 5450, 5628, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=39026 dup_rows=21096 dup_rows_remote=18628 dedup_ratio=0.321899 router_tokens_per_expert=[550, 492, 491, 536, 497, 508, 549, 517, 522, 504, 498, 501, 490, 492, 498, 516, 522, 514, 493, 546, 518, 500, 560, 517, 512, 543, 517, 498, 511, 486, 483, 474, 527, 540, 533, 514, 537, 523, 519, 520, 479, 512, 499, 494, 529, 492, 450, 486, 495, 487, 564, 496, 515, 520, 504, 507, 542, 492, 544, 509, 545, 529, 486, 524] payload_rows_per_dst=[5600, 5472, 5546, 5528, 5646, 5414, 5518, 5716] dup_rows_per_dst=[2680, 2570, 2794, 2520, 2780, 2468, 2658, 2626] combine_rows_per_src=[5600, 5472, 5546, 5528, 5646, 5414, 5518, 5716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44552 payload_rows_remote=38958 dup_rows=20984 dup_rows_remote=18256 dedup_ratio=0.320190 router_tokens_per_expert=[484, 505, 480, 529, 473, 518, 519, 530, 519, 529, 507, 540, 531, 494, 503, 538, 496, 497, 534, 496, 511, 518, 580, 548, 486, 500, 511, 513, 504, 495, 490, 506, 498, 497, 522, 519, 534, 517, 537, 515, 543, 489, 533, 489, 485, 494, 520, 518, 502, 526, 495, 506, 504, 504, 512, 511, 495, 519, 529, 529, 522, 491, 532, 497] payload_rows_per_dst=[5540, 5594, 5682, 5522, 5650, 5478, 5470, 5616] dup_rows_per_dst=[2536, 2728, 2678, 2488, 2628, 2664, 2650, 2612] combine_rows_per_src=[5540, 5594, 5682, 5522, 5650, 5478, 5470, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158801920 dispatch_meta_bytes=229240 combine_payload_bytes=158801920 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159031160 ingress_bytes=209133568 link_bytes=209133568 token_bytes=317833080 weight_movement_bytes=50331648 total_nvlink_bytes=368164728 weight_to_token_ratio=0.158359 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591264812e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158515200 dispatch_meta_bytes=228840 combine_payload_bytes=158515200 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158744040 ingress_bytes=208846848 link_bytes=208846848 token_bytes=317259240 weight_movement_bytes=50331648 total_nvlink_bytes=367590888 weight_to_token_ratio=0.158645 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589083204e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44382 payload_rows_remote=38792 dup_rows=21154 dup_rows_remote=18392 dedup_ratio=0.322784 router_tokens_per_expert=[502, 489, 491, 503, 501, 515, 494, 488, 524, 525, 535, 540, 509, 457, 507, 514, 476, 470, 505, 527, 527, 514, 503, 505, 526, 530, 502, 516, 549, 486, 497, 509, 511, 453, 526, 533, 547, 512, 514, 501, 528, 520, 502, 536, 488, 508, 504, 499, 527, 505, 505, 526, 558, 501, 538, 546, 517, 496, 518, 502, 522, 527, 528, 534] payload_rows_per_dst=[5590, 5416, 5486, 5632, 5498, 5674, 5600, 5486] dup_rows_per_dst=[2762, 2508, 2568, 2800, 2494, 2800, 2724, 2498] combine_rows_per_src=[5590, 5416, 5486, 5632, 5498, 5674, 5600, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159449088 dispatch_meta_bytes=228880 combine_payload_bytes=159449088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159677968 ingress_bytes=184614912 link_bytes=184614912 token_bytes=319127056 weight_movement_bytes=25165824 total_nvlink_bytes=344292880 weight_to_token_ratio=0.078858 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404706170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159850496 dispatch_meta_bytes=230616 combine_payload_bytes=159850496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160081112 ingress_bytes=185016320 link_bytes=185016320 token_bytes=319931608 weight_movement_bytes=25165824 total_nvlink_bytes=345097432 weight_to_token_ratio=0.078660 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407760421e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159571968 dispatch_meta_bytes=228856 combine_payload_bytes=159571968 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159800824 ingress_bytes=209903616 link_bytes=209903616 token_bytes=319372792 weight_movement_bytes=50331648 total_nvlink_bytes=369704440 weight_to_token_ratio=0.157595 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597123988e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44398 payload_rows_remote=38910 dup_rows=21138 dup_rows_remote=18490 dedup_ratio=0.322540 router_tokens_per_expert=[477, 505, 547, 479, 526, 502, 470, 530, 557, 503, 520, 480, 525, 474, 564, 513, 527, 529, 501, 530, 509, 504, 512, 499, 490, 516, 508, 503, 510, 529, 498, 522, 523, 526, 490, 537, 546, 510, 533, 474, 528, 490, 516, 498, 510, 513, 505, 517, 501, 526, 517, 499, 489, 479, 525, 532, 533, 523, 488, 492, 549, 536, 508, 496] payload_rows_per_dst=[5424, 5652, 5570, 5590, 5582, 5500, 5488, 5592] dup_rows_per_dst=[2648, 2620, 2652, 2562, 2696, 2654, 2648, 2658] combine_rows_per_src=[5424, 5652, 5570, 5590, 5582, 5500, 5488, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44506 payload_rows_remote=38804 dup_rows=21030 dup_rows_remote=18026 dedup_ratio=0.320892 router_tokens_per_expert=[474, 528, 488, 480, 517, 534, 527, 496, 481, 546, 509, 500, 547, 513, 486, 467, 514, 536, 515, 509, 507, 499, 538, 510, 476, 509, 519, 529, 500, 502, 493, 505, 519, 509, 505, 529, 517, 503, 486, 486, 476, 495, 475, 518, 540, 547, 526, 545, 549, 532, 497, 527, 575, 496, 501, 498, 505, 510, 520, 542, 516, 500, 569, 501] payload_rows_per_dst=[5532, 5502, 5604, 5536, 5514, 5506, 5610, 5702] dup_rows_per_dst=[2556, 2596, 2652, 2402, 2594, 2486, 2740, 3004] combine_rows_per_src=[5532, 5502, 5604, 5536, 5514, 5506, 5610, 5702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158892032 dispatch_meta_bytes=228736 combine_payload_bytes=158892032 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159120768 ingress_bytes=184057856 link_bytes=184057856 token_bytes=318012800 weight_movement_bytes=25165824 total_nvlink_bytes=343178624 weight_to_token_ratio=0.079135 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400467617e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44248 payload_rows_remote=38688 dup_rows=21288 dup_rows_remote=18704 dedup_ratio=0.324829 router_tokens_per_expert=[533, 491, 498, 469, 496, 505, 508, 515, 562, 500, 526, 512, 548, 532, 520, 478, 495, 534, 500, 522, 498, 513, 518, 532, 493, 500, 533, 512, 518, 550, 494, 531, 510, 501, 524, 510, 495, 530, 495, 507, 503, 523, 548, 508, 532, 504, 505, 537, 528, 501, 507, 486, 526, 503, 501, 480, 511, 500, 461, 513, 519, 540, 514, 510] payload_rows_per_dst=[5438, 5634, 5570, 5558, 5560, 5494, 5442, 5552] dup_rows_per_dst=[2592, 2722, 2654, 2704, 2584, 2826, 2622, 2584] combine_rows_per_src=[5438, 5634, 5570, 5558, 5560, 5494, 5442, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159375360 dispatch_meta_bytes=229600 combine_payload_bytes=159375360 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159604960 ingress_bytes=234872832 link_bytes=234872832 token_bytes=318980320 weight_movement_bytes=75497472 total_nvlink_bytes=394477792 weight_to_token_ratio=0.236684 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787110871e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=227320 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159168504 ingress_bytes=209272832 link_bytes=209272832 token_bytes=318109688 weight_movement_bytes=50331648 total_nvlink_bytes=368441336 weight_to_token_ratio=0.158221 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592324450e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=39048 dup_rows=21000 dup_rows_remote=18390 dedup_ratio=0.320435 router_tokens_per_expert=[494, 501, 488, 493, 531, 506, 521, 539, 521, 475, 533, 487, 479, 516, 476, 496, 530, 527, 494, 502, 503, 488, 501, 504, 542, 504, 531, 496, 533, 480, 537, 516, 517, 549, 508, 491, 526, 496, 515, 501, 546, 541, 507, 546, 506, 509, 534, 494, 498, 483, 510, 499, 505, 511, 551, 522, 513, 494, 530, 528, 515, 545, 528, 506] payload_rows_per_dst=[5612, 5442, 5488, 5574, 5568, 5656, 5592, 5604] dup_rows_per_dst=[2534, 2524, 2610, 2704, 2638, 2710, 2566, 2714] combine_rows_per_src=[5612, 5442, 5488, 5574, 5568, 5656, 5592, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44570 payload_rows_remote=38908 dup_rows=20966 dup_rows_remote=18242 dedup_ratio=0.319916 router_tokens_per_expert=[507, 531, 488, 498, 492, 474, 497, 517, 528, 534, 488, 488, 502, 532, 522, 517, 492, 489, 505, 518, 478, 497, 514, 488, 524, 546, 481, 529, 560, 528, 514, 511, 529, 503, 514, 525, 515, 517, 543, 507, 485, 502, 500, 519, 512, 494, 493, 522, 532, 499, 546, 524, 494, 510, 512, 529, 533, 493, 511, 510, 532, 517, 520, 537] payload_rows_per_dst=[5524, 5574, 5492, 5662, 5600, 5508, 5628, 5582] dup_rows_per_dst=[2484, 2648, 2470, 2724, 2706, 2546, 2664, 2724] combine_rows_per_src=[5524, 5574, 5492, 5662, 5600, 5508, 5628, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=39040 dup_rows=21000 dup_rows_remote=18262 dedup_ratio=0.320435 router_tokens_per_expert=[492, 554, 532, 518, 531, 508, 457, 501, 500, 510, 517, 518, 489, 536, 538, 513, 506, 491, 537, 513, 534, 487, 503, 517, 526, 499, 499, 493, 482, 503, 514, 490, 492, 497, 527, 529, 498, 488, 558, 497, 554, 521, 512, 462, 515, 491, 517, 545, 528, 540, 555, 546, 517, 525, 490, 487, 486, 497, 517, 524, 534, 511, 513, 487] payload_rows_per_dst=[5586, 5684, 5530, 5444, 5554, 5496, 5654, 5588] dup_rows_per_dst=[2600, 2558, 2646, 2568, 2618, 2738, 2722, 2550] combine_rows_per_src=[5586, 5684, 5530, 5444, 5554, 5496, 5654, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44732 payload_rows_remote=39104 dup_rows=20804 dup_rows_remote=18224 dedup_ratio=0.317444 router_tokens_per_expert=[507, 515, 492, 515, 520, 485, 482, 509, 525, 514, 546, 528, 486, 483, 519, 503, 517, 493, 501, 484, 522, 535, 525, 520, 517, 510, 484, 519, 505, 538, 536, 499, 534, 529, 505, 466, 499, 541, 547, 544, 513, 493, 531, 521, 494, 505, 530, 544, 533, 480, 488, 513, 550, 475, 493, 506, 510, 524, 484, 513, 499, 510, 520, 540] payload_rows_per_dst=[5576, 5628, 5538, 5546, 5642, 5672, 5544, 5586] dup_rows_per_dst=[2474, 2580, 2656, 2670, 2688, 2590, 2532, 2614] combine_rows_per_src=[5576, 5628, 5538, 5546, 5642, 5672, 5544, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158466048 dispatch_meta_bytes=229568 combine_payload_bytes=158466048 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158695616 ingress_bytes=208797696 link_bytes=208797696 token_bytes=317161664 weight_movement_bytes=50331648 total_nvlink_bytes=367493312 weight_to_token_ratio=0.158694 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588709214e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44452 payload_rows_remote=38562 dup_rows=21084 dup_rows_remote=18018 dedup_ratio=0.321716 router_tokens_per_expert=[511, 519, 542, 507, 514, 494, 495, 520, 505, 500, 514, 495, 501, 519, 518, 518, 487, 506, 519, 468, 500, 517, 466, 511, 531, 536, 485, 512, 516, 513, 498, 541, 546, 515, 508, 522, 499, 544, 514, 491, 480, 539, 518, 488, 495, 523, 499, 535, 505, 499, 526, 500, 512, 528, 555, 532, 518, 530, 534, 504, 508, 520, 490, 513] payload_rows_per_dst=[5890, 5480, 5674, 5606, 5434, 5544, 5540, 5284] dup_rows_per_dst=[3066, 2692, 2730, 2686, 2558, 2610, 2544, 2198] combine_rows_per_src=[5890, 5480, 5674, 5606, 5434, 5544, 5540, 5284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159367168 dispatch_meta_bytes=228600 combine_payload_bytes=159367168 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159595768 ingress_bytes=209698816 link_bytes=209698816 token_bytes=318962936 weight_movement_bytes=50331648 total_nvlink_bytes=369294584 weight_to_token_ratio=0.157798 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595565696e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159940608 dispatch_meta_bytes=229752 combine_payload_bytes=159940608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160170360 ingress_bytes=185106432 link_bytes=185106432 token_bytes=320110968 weight_movement_bytes=25165824 total_nvlink_bytes=345276792 weight_to_token_ratio=0.078616 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408446069e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159907840 dispatch_meta_bytes=229208 combine_payload_bytes=159907840 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160137048 ingress_bytes=210239488 link_bytes=210239488 token_bytes=320044888 weight_movement_bytes=50331648 total_nvlink_bytes=370376536 weight_to_token_ratio=0.157264 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599679585e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160169984 dispatch_meta_bytes=229312 combine_payload_bytes=160169984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160399296 ingress_bytes=185335808 link_bytes=185335808 token_bytes=320569280 weight_movement_bytes=25165824 total_nvlink_bytes=345735104 weight_to_token_ratio=0.078504 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410191355e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38782 dup_rows=21116 dup_rows_remote=18176 dedup_ratio=0.322205 router_tokens_per_expert=[503, 543, 510, 505, 521, 469, 515, 502, 498, 529, 533, 502, 524, 520, 546, 507, 531, 546, 525, 484, 530, 525, 522, 513, 494, 483, 542, 518, 538, 488, 487, 522, 522, 496, 502, 475, 509, 493, 535, 459, 494, 535, 493, 503, 515, 519, 516, 525, 496, 523, 520, 541, 511, 470, 525, 498, 520, 511, 508, 544, 492, 555, 472, 516] payload_rows_per_dst=[5414, 5636, 5454, 5602, 5576, 5572, 5528, 5638] dup_rows_per_dst=[2490, 2666, 2372, 2660, 2638, 2694, 2656, 2940] combine_rows_per_src=[5414, 5636, 5454, 5602, 5576, 5572, 5528, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44506 payload_rows_remote=38946 dup_rows=21030 dup_rows_remote=18416 dedup_ratio=0.320892 router_tokens_per_expert=[509, 502, 526, 494, 542, 503, 485, 492, 521, 522, 505, 483, 487, 493, 523, 499, 516, 519, 524, 510, 518, 498, 535, 508, 535, 510, 500, 523, 527, 498, 548, 516, 508, 486, 502, 494, 517, 492, 534, 506, 510, 552, 501, 530, 517, 510, 525, 488, 539, 501, 508, 488, 557, 496, 503, 495, 561, 479, 516, 521, 497, 526, 517, 521] payload_rows_per_dst=[5580, 5502, 5550, 5618, 5506, 5616, 5560, 5574] dup_rows_per_dst=[2526, 2564, 2706, 2696, 2572, 2650, 2614, 2702] combine_rows_per_src=[5580, 5502, 5550, 5618, 5506, 5616, 5560, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=157949952 dispatch_meta_bytes=226320 combine_payload_bytes=157949952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158176272 ingress_bytes=183115776 link_bytes=183115776 token_bytes=316126224 weight_movement_bytes=25165824 total_nvlink_bytes=341292048 weight_to_token_ratio=0.079607 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.393299477e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158851072 dispatch_meta_bytes=227832 combine_payload_bytes=158851072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159078904 ingress_bytes=184016896 link_bytes=184016896 token_bytes=317929976 weight_movement_bytes=25165824 total_nvlink_bytes=343095800 weight_to_token_ratio=0.079155 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400155959e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229448 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159752264 ingress_bytes=235020288 link_bytes=235020288 token_bytes=319275080 weight_movement_bytes=75497472 total_nvlink_bytes=394772552 weight_to_token_ratio=0.236465 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788232841e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44480 payload_rows_remote=38868 dup_rows=21056 dup_rows_remote=18450 dedup_ratio=0.321289 router_tokens_per_expert=[498, 531, 483, 507, 502, 519, 525, 486, 507, 523, 508, 490, 499, 533, 523, 493, 528, 534, 536, 483, 544, 476, 546, 536, 502, 500, 540, 517, 519, 511, 508, 495, 510, 538, 505, 494, 513, 506, 512, 531, 534, 546, 485, 478, 528, 524, 469, 499, 538, 477, 542, 546, 550, 510, 486, 513, 504, 520, 531, 556, 476, 481, 467, 497] payload_rows_per_dst=[5560, 5552, 5620, 5558, 5612, 5512, 5626, 5440] dup_rows_per_dst=[2542, 2600, 2746, 2626, 2606, 2614, 2698, 2624] combine_rows_per_src=[5560, 5552, 5620, 5558, 5612, 5512, 5626, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44374 payload_rows_remote=38762 dup_rows=21162 dup_rows_remote=18548 dedup_ratio=0.322906 router_tokens_per_expert=[520, 516, 525, 507, 501, 550, 547, 504, 514, 507, 509, 529, 516, 513, 515, 532, 483, 498, 521, 487, 535, 499, 545, 508, 525, 553, 527, 515, 531, 494, 453, 515, 500, 498, 496, 527, 530, 498, 544, 509, 542, 530, 486, 446, 521, 491, 512, 508, 522, 470, 526, 484, 501, 534, 536, 542, 482, 501, 551, 493, 487, 516, 494, 497] payload_rows_per_dst=[5608, 5528, 5552, 5612, 5524, 5492, 5542, 5516] dup_rows_per_dst=[2732, 2742, 2600, 2614, 2680, 2580, 2688, 2526] combine_rows_per_src=[5608, 5528, 5552, 5612, 5524, 5492, 5542, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=38960 dup_rows=20978 dup_rows_remote=18278 dedup_ratio=0.320099 router_tokens_per_expert=[516, 535, 495, 497, 541, 491, 503, 545, 500, 529, 532, 519, 508, 524, 506, 513, 454, 523, 516, 539, 527, 509, 484, 539, 539, 513, 485, 487, 523, 508, 505, 514, 502, 513, 505, 521, 540, 528, 540, 496, 485, 544, 507, 518, 497, 550, 493, 555, 508, 477, 513, 496, 511, 534, 487, 509, 508, 495, 522, 500, 484, 490, 538, 483] payload_rows_per_dst=[5654, 5626, 5542, 5548, 5576, 5598, 5546, 5468] dup_rows_per_dst=[2592, 2636, 2640, 2600, 2714, 2700, 2524, 2572] combine_rows_per_src=[5654, 5626, 5542, 5548, 5576, 5598, 5546, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44410 payload_rows_remote=38776 dup_rows=21126 dup_rows_remote=18404 dedup_ratio=0.322357 router_tokens_per_expert=[505, 515, 535, 534, 517, 532, 523, 473, 505, 494, 505, 536, 520, 513, 510, 518, 526, 543, 491, 520, 536, 510, 539, 513, 471, 527, 492, 491, 499, 540, 510, 509, 536, 525, 521, 460, 502, 519, 492, 514, 514, 496, 509, 514, 510, 478, 496, 519, 531, 530, 530, 487, 514, 511, 512, 506, 504, 510, 493, 491, 529, 503, 516, 544] payload_rows_per_dst=[5534, 5588, 5634, 5500, 5548, 5460, 5606, 5540] dup_rows_per_dst=[2734, 2614, 2722, 2578, 2590, 2612, 2636, 2640] combine_rows_per_src=[5534, 5588, 5634, 5500, 5548, 5460, 5606, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39102 dup_rows=20934 dup_rows_remote=18296 dedup_ratio=0.319427 router_tokens_per_expert=[510, 534, 529, 513, 552, 531, 497, 497, 501, 537, 481, 540, 479, 514, 521, 496, 531, 540, 499, 537, 481, 532, 500, 514, 501, 515, 502, 510, 461, 549, 526, 491, 542, 532, 521, 493, 536, 507, 481, 470, 486, 512, 505, 496, 481, 560, 534, 501, 473, 511, 519, 497, 491, 530, 523, 502, 518, 510, 515, 532, 549, 511, 503, 506] payload_rows_per_dst=[5664, 5500, 5654, 5528, 5536, 5586, 5526, 5608] dup_rows_per_dst=[2662, 2638, 2614, 2582, 2628, 2564, 2566, 2680] combine_rows_per_src=[5664, 5500, 5654, 5528, 5536, 5586, 5526, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159203328 dispatch_meta_bytes=229272 combine_payload_bytes=159203328 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159432600 ingress_bytes=184369152 link_bytes=184369152 token_bytes=318635928 weight_movement_bytes=25165824 total_nvlink_bytes=343801752 weight_to_token_ratio=0.078980 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402836220e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158769152 dispatch_meta_bytes=229240 combine_payload_bytes=158769152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158998392 ingress_bytes=183934976 link_bytes=183934976 token_bytes=317767544 weight_movement_bytes=25165824 total_nvlink_bytes=342933368 weight_to_token_ratio=0.079196 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399532642e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38868 dup_rows=21076 dup_rows_remote=18424 dedup_ratio=0.321594 router_tokens_per_expert=[576, 536, 481, 460, 511, 506, 489, 526, 513, 552, 499, 474, 485, 514, 535, 500, 533, 500, 479, 537, 486, 480, 517, 514, 528, 526, 510, 513, 490, 542, 507, 491, 532, 493, 527, 508, 558, 485, 507, 505, 508, 531, 506, 509, 501, 537, 521, 530, 532, 519, 515, 537, 507, 526, 511, 519, 498, 478, 517, 488, 516, 516, 529, 492] payload_rows_per_dst=[5592, 5582, 5556, 5488, 5588, 5646, 5548, 5460] dup_rows_per_dst=[2652, 2562, 2574, 2726, 2642, 2640, 2672, 2608] combine_rows_per_src=[5592, 5582, 5556, 5488, 5588, 5646, 5548, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159580160 dispatch_meta_bytes=228952 combine_payload_bytes=159580160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159809112 ingress_bytes=184745984 link_bytes=184745984 token_bytes=319389272 weight_movement_bytes=25165824 total_nvlink_bytes=344555096 weight_to_token_ratio=0.078794 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405703476e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=228720 combine_payload_bytes=158826496 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159055216 ingress_bytes=259489792 link_bytes=259489792 token_bytes=317881712 weight_movement_bytes=100663296 total_nvlink_bytes=418545008 weight_to_token_ratio=0.316669 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.974417493e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160161792 dispatch_meta_bytes=229592 combine_payload_bytes=160161792 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160391384 ingress_bytes=185327616 link_bytes=185327616 token_bytes=320553176 weight_movement_bytes=25165824 total_nvlink_bytes=345719000 weight_to_token_ratio=0.078507 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410129024e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44372 payload_rows_remote=38728 dup_rows=21164 dup_rows_remote=18464 dedup_ratio=0.322937 router_tokens_per_expert=[489, 490, 507, 501, 494, 519, 479, 514, 565, 482, 527, 498, 508, 466, 551, 565, 471, 535, 507, 506, 538, 516, 494, 530, 524, 484, 525, 503, 493, 514, 532, 529, 506, 514, 525, 499, 496, 531, 516, 532, 505, 500, 538, 549, 501, 523, 495, 515, 494, 503, 499, 479, 477, 496, 508, 539, 505, 515, 530, 510, 507, 549, 513, 543] payload_rows_per_dst=[5496, 5578, 5472, 5556, 5602, 5592, 5432, 5644] dup_rows_per_dst=[2490, 2746, 2722, 2652, 2636, 2660, 2558, 2700] combine_rows_per_src=[5496, 5578, 5472, 5556, 5602, 5592, 5432, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=39022 dup_rows=21012 dup_rows_remote=18308 dedup_ratio=0.320618 router_tokens_per_expert=[527, 501, 526, 489, 483, 518, 509, 571, 515, 513, 520, 480, 533, 522, 506, 487, 506, 530, 527, 529, 523, 494, 514, 510, 513, 485, 507, 517, 510, 509, 520, 467, 536, 526, 514, 502, 517, 499, 515, 490, 530, 500, 497, 521, 522, 530, 516, 500, 508, 502, 489, 499, 564, 508, 506, 527, 521, 509, 535, 502, 471, 492, 559, 500] payload_rows_per_dst=[5590, 5602, 5604, 5508, 5556, 5590, 5502, 5572] dup_rows_per_dst=[2658, 2550, 2662, 2548, 2642, 2642, 2704, 2606] combine_rows_per_src=[5590, 5602, 5604, 5508, 5556, 5590, 5502, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159203328 dispatch_meta_bytes=229168 combine_payload_bytes=159203328 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159432496 ingress_bytes=209534976 link_bytes=209534976 token_bytes=318635824 weight_movement_bytes=50331648 total_nvlink_bytes=368967472 weight_to_token_ratio=0.157960 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594319063e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158629888 dispatch_meta_bytes=228768 combine_payload_bytes=158629888 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=158858656 ingress_bytes=284459008 link_bytes=284459008 token_bytes=317488544 weight_movement_bytes=125829120 total_nvlink_bytes=443317664 weight_to_token_ratio=0.396326 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.164404377e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44368 payload_rows_remote=38806 dup_rows=21168 dup_rows_remote=18474 dedup_ratio=0.322998 router_tokens_per_expert=[540, 546, 496, 522, 463, 542, 549, 481, 512, 500, 495, 560, 532, 513, 548, 491, 515, 531, 499, 474, 525, 527, 496, 511, 546, 477, 513, 507, 524, 482, 542, 500, 528, 517, 504, 488, 573, 505, 504, 509, 489, 503, 487, 527, 509, 516, 511, 501, 523, 553, 466, 480, 523, 501, 555, 494, 505, 501, 529, 491, 506, 509, 513, 489] payload_rows_per_dst=[5588, 5640, 5504, 5550, 5562, 5504, 5542, 5478] dup_rows_per_dst=[2690, 2662, 2652, 2632, 2694, 2582, 2648, 2608] combine_rows_per_src=[5588, 5640, 5504, 5550, 5562, 5504, 5542, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159834112 dispatch_meta_bytes=229320 combine_payload_bytes=159834112 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160063432 ingress_bytes=210165760 link_bytes=210165760 token_bytes=319897544 weight_movement_bytes=50331648 total_nvlink_bytes=370229192 weight_to_token_ratio=0.157337 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599118600e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44748 payload_rows_remote=39078 dup_rows=20788 dup_rows_remote=18150 dedup_ratio=0.317200 router_tokens_per_expert=[496, 514, 512, 534, 490, 518, 516, 499, 478, 503, 475, 524, 510, 536, 503, 522, 510, 491, 481, 500, 479, 536, 527, 503, 511, 532, 489, 511, 525, 520, 536, 530, 522, 513, 497, 513, 492, 525, 467, 516, 524, 552, 481, 525, 523, 508, 491, 507, 504, 512, 490, 492, 575, 536, 555, 499, 522, 556, 516, 552, 484, 507, 496, 505] payload_rows_per_dst=[5548, 5636, 5524, 5670, 5556, 5618, 5568, 5628] dup_rows_per_dst=[2610, 2466, 2530, 2638, 2534, 2604, 2758, 2648] combine_rows_per_src=[5548, 5636, 5524, 5670, 5556, 5618, 5568, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38974 dup_rows=21052 dup_rows_remote=18494 dedup_ratio=0.321228 router_tokens_per_expert=[513, 527, 487, 506, 482, 503, 513, 501, 531, 509, 488, 508, 543, 510, 512, 511, 500, 533, 497, 499, 527, 501, 541, 511, 529, 516, 522, 532, 531, 503, 518, 521, 489, 495, 515, 517, 474, 505, 553, 500, 499, 487, 480, 504, 512, 493, 526, 533, 531, 503, 526, 533, 513, 516, 493, 529, 515, 517, 495, 501, 544, 510, 507, 528] payload_rows_per_dst=[5518, 5568, 5530, 5586, 5588, 5510, 5652, 5532] dup_rows_per_dst=[2546, 2656, 2688, 2758, 2508, 2558, 2636, 2702] combine_rows_per_src=[5518, 5568, 5530, 5586, 5588, 5510, 5652, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44772 payload_rows_remote=39206 dup_rows=20764 dup_rows_remote=18334 dedup_ratio=0.316833 router_tokens_per_expert=[488, 492, 529, 527, 550, 495, 470, 495, 502, 529, 490, 522, 488, 527, 505, 488, 475, 497, 490, 504, 501, 518, 520, 493, 531, 514, 543, 511, 510, 497, 492, 512, 528, 497, 506, 584, 505, 542, 530, 517, 540, 500, 492, 494, 537, 507, 538, 518, 511, 534, 545, 515, 532, 529, 498, 485, 521, 516, 509, 482, 474, 510, 515, 552] payload_rows_per_dst=[5544, 5554, 5566, 5592, 5670, 5584, 5648, 5614] dup_rows_per_dst=[2548, 2548, 2430, 2628, 2748, 2668, 2650, 2544] combine_rows_per_src=[5544, 5554, 5566, 5592, 5670, 5584, 5648, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=39016 dup_rows=20978 dup_rows_remote=18338 dedup_ratio=0.320099 router_tokens_per_expert=[489, 528, 507, 505, 522, 506, 554, 495, 529, 551, 509, 493, 524, 515, 508, 462, 542, 486, 528, 518, 495, 491, 481, 499, 560, 521, 511, 498, 511, 514, 509, 507, 506, 519, 522, 546, 473, 480, 522, 495, 508, 509, 481, 521, 533, 503, 556, 527, 513, 526, 544, 473, 524, 530, 524, 497, 512, 490, 516, 535, 500, 520, 518, 477] payload_rows_per_dst=[5636, 5542, 5542, 5550, 5544, 5574, 5642, 5528] dup_rows_per_dst=[2576, 2640, 2538, 2712, 2582, 2702, 2620, 2608] combine_rows_per_src=[5636, 5542, 5542, 5550, 5544, 5574, 5642, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158949376 dispatch_meta_bytes=229120 combine_payload_bytes=158949376 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159178496 ingress_bytes=184115200 link_bytes=184115200 token_bytes=318127872 weight_movement_bytes=25165824 total_nvlink_bytes=343293696 weight_to_token_ratio=0.079106 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400903939e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=228912 combine_payload_bytes=160063488 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160292400 ingress_bytes=210395136 link_bytes=210395136 token_bytes=320355888 weight_movement_bytes=50331648 total_nvlink_bytes=370687536 weight_to_token_ratio=0.157112 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600863887e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38724 dup_rows=21166 dup_rows_remote=18314 dedup_ratio=0.322968 router_tokens_per_expert=[520, 526, 533, 487, 495, 491, 514, 524, 483, 519, 536, 496, 511, 517, 481, 529, 533, 488, 506, 497, 497, 544, 505, 506, 505, 516, 529, 487, 514, 518, 529, 536, 524, 477, 546, 525, 530, 468, 484, 492, 473, 504, 493, 499, 511, 502, 506, 516, 530, 515, 517, 538, 495, 538, 542, 474, 514, 542, 517, 528, 514, 518, 543, 521] payload_rows_per_dst=[5646, 5584, 5624, 5446, 5484, 5520, 5494, 5572] dup_rows_per_dst=[2852, 2564, 2948, 2510, 2630, 2628, 2482, 2552] combine_rows_per_src=[5646, 5584, 5624, 5446, 5484, 5520, 5494, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159637504 dispatch_meta_bytes=229872 combine_payload_bytes=159637504 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159867376 ingress_bytes=184803328 link_bytes=184803328 token_bytes=319504880 weight_movement_bytes=25165824 total_nvlink_bytes=344670704 weight_to_token_ratio=0.078765 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406139798e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38912 dup_rows=21100 dup_rows_remote=18522 dedup_ratio=0.321960 router_tokens_per_expert=[519, 527, 522, 547, 505, 537, 456, 518, 516, 533, 505, 490, 503, 504, 519, 528, 529, 489, 503, 512, 537, 492, 525, 493, 524, 522, 506, 503, 500, 540, 504, 541, 484, 476, 544, 546, 520, 529, 532, 506, 501, 516, 528, 475, 508, 547, 497, 521, 497, 519, 485, 533, 507, 503, 499, 511, 534, 489, 538, 494, 467, 522, 497, 494] payload_rows_per_dst=[5634, 5514, 5532, 5602, 5554, 5556, 5520, 5524] dup_rows_per_dst=[2628, 2682, 2628, 2678, 2720, 2598, 2588, 2578] combine_rows_per_src=[5634, 5514, 5532, 5602, 5554, 5556, 5520, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160587776 dispatch_meta_bytes=230160 combine_payload_bytes=160587776 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160817936 ingress_bytes=185753600 link_bytes=185753600 token_bytes=321405712 weight_movement_bytes=25165824 total_nvlink_bytes=346571536 weight_to_token_ratio=0.078299 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.413370270e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159809536 dispatch_meta_bytes=229416 combine_payload_bytes=159809536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160038952 ingress_bytes=184975360 link_bytes=184975360 token_bytes=319848488 weight_movement_bytes=25165824 total_nvlink_bytes=345014312 weight_to_token_ratio=0.078680 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407448762e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44574 payload_rows_remote=38990 dup_rows=20962 dup_rows_remote=18346 dedup_ratio=0.319855 router_tokens_per_expert=[492, 504, 520, 487, 511, 514, 515, 515, 521, 559, 499, 527, 503, 499, 512, 499, 474, 516, 501, 531, 491, 522, 535, 547, 531, 497, 507, 523, 504, 489, 535, 505, 541, 491, 518, 533, 495, 484, 526, 546, 548, 521, 495, 476, 489, 515, 511, 496, 539, 552, 523, 534, 478, 502, 481, 491, 522, 545, 478, 495, 525, 540, 507, 486] payload_rows_per_dst=[5482, 5576, 5542, 5570, 5618, 5600, 5584, 5602] dup_rows_per_dst=[2634, 2662, 2692, 2612, 2650, 2502, 2616, 2594] combine_rows_per_src=[5482, 5576, 5542, 5570, 5618, 5600, 5584, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158613504 dispatch_meta_bytes=228152 combine_payload_bytes=158613504 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158841656 ingress_bytes=183779328 link_bytes=183779328 token_bytes=317455160 weight_movement_bytes=25165824 total_nvlink_bytes=342620984 weight_to_token_ratio=0.079274 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398348341e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159383552 dispatch_meta_bytes=229736 combine_payload_bytes=159383552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159613288 ingress_bytes=209715200 link_bytes=209715200 token_bytes=318996840 weight_movement_bytes=50331648 total_nvlink_bytes=369328488 weight_to_token_ratio=0.157781 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595690359e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159703040 dispatch_meta_bytes=229344 combine_payload_bytes=159703040 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159932384 ingress_bytes=184868864 link_bytes=184868864 token_bytes=319635424 weight_movement_bytes=25165824 total_nvlink_bytes=344801248 weight_to_token_ratio=0.078733 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406638451e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38968 dup_rows=21036 dup_rows_remote=18404 dedup_ratio=0.320984 router_tokens_per_expert=[491, 500, 502, 498, 481, 526, 486, 543, 517, 461, 533, 538, 520, 476, 525, 537, 535, 502, 493, 523, 509, 510, 534, 508, 478, 526, 492, 471, 527, 512, 512, 520, 513, 522, 502, 498, 529, 524, 487, 507, 509, 529, 535, 512, 531, 496, 516, 492, 479, 536, 493, 533, 497, 494, 524, 521, 526, 519, 525, 533, 510, 535, 517, 538] payload_rows_per_dst=[5520, 5536, 5594, 5526, 5532, 5562, 5588, 5642] dup_rows_per_dst=[2534, 2678, 2634, 2550, 2632, 2678, 2566, 2764] combine_rows_per_src=[5520, 5536, 5594, 5526, 5532, 5562, 5588, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44304 payload_rows_remote=38798 dup_rows=21232 dup_rows_remote=18650 dedup_ratio=0.323975 router_tokens_per_expert=[477, 562, 495, 474, 509, 519, 497, 501, 509, 472, 535, 513, 501, 546, 523, 506, 460, 536, 526, 519, 513, 501, 514, 541, 517, 445, 504, 505, 545, 470, 535, 523, 537, 511, 514, 496, 508, 519, 478, 482, 553, 492, 527, 518, 547, 527, 514, 503, 517, 496, 539, 481, 500, 492, 524, 514, 560, 544, 525, 498, 524, 509, 515, 511] payload_rows_per_dst=[5438, 5532, 5524, 5506, 5524, 5574, 5542, 5664] dup_rows_per_dst=[2630, 2678, 2696, 2582, 2566, 2788, 2584, 2708] combine_rows_per_src=[5438, 5532, 5524, 5506, 5524, 5574, 5542, 5664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44362 payload_rows_remote=38764 dup_rows=21174 dup_rows_remote=18572 dedup_ratio=0.323090 router_tokens_per_expert=[573, 479, 509, 525, 532, 478, 543, 513, 496, 498, 543, 522, 520, 491, 510, 501, 534, 498, 506, 524, 491, 535, 507, 497, 540, 545, 522, 481, 510, 488, 465, 516, 556, 512, 492, 528, 497, 538, 514, 517, 510, 539, 470, 504, 524, 517, 512, 524, 509, 491, 525, 489, 534, 537, 497, 501, 527, 517, 508, 478, 525, 499, 490, 495] payload_rows_per_dst=[5554, 5532, 5524, 5534, 5582, 5598, 5518, 5520] dup_rows_per_dst=[2750, 2630, 2660, 2600, 2726, 2602, 2648, 2558] combine_rows_per_src=[5554, 5532, 5524, 5534, 5582, 5598, 5518, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44470 payload_rows_remote=38918 dup_rows=21066 dup_rows_remote=18528 dedup_ratio=0.321442 router_tokens_per_expert=[523, 537, 483, 480, 506, 505, 557, 497, 505, 509, 528, 504, 516, 550, 527, 501, 483, 465, 522, 530, 519, 501, 522, 503, 522, 527, 553, 509, 539, 497, 500, 543, 495, 517, 487, 476, 497, 542, 522, 497, 484, 511, 510, 515, 472, 540, 523, 516, 496, 514, 555, 521, 534, 527, 480, 500, 510, 503, 506, 537, 522, 500, 486, 510] payload_rows_per_dst=[5520, 5710, 5552, 5616, 5504, 5504, 5546, 5518] dup_rows_per_dst=[2656, 2570, 2538, 2764, 2562, 2638, 2708, 2630] combine_rows_per_src=[5520, 5710, 5552, 5616, 5504, 5504, 5546, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=39008 dup_rows=20946 dup_rows_remote=18454 dedup_ratio=0.319611 router_tokens_per_expert=[507, 533, 512, 533, 510, 495, 517, 505, 477, 467, 545, 488, 521, 508, 524, 507, 495, 514, 511, 529, 496, 493, 517, 484, 521, 508, 498, 497, 533, 550, 484, 564, 503, 502, 520, 528, 516, 530, 510, 540, 498, 484, 524, 549, 520, 501, 496, 499, 514, 529, 518, 523, 492, 482, 470, 509, 528, 498, 535, 508, 522, 536, 516, 525] payload_rows_per_dst=[5598, 5582, 5548, 5628, 5570, 5478, 5522, 5664] dup_rows_per_dst=[2626, 2492, 2530, 2682, 2728, 2664, 2552, 2672] combine_rows_per_src=[5598, 5582, 5548, 5628, 5570, 5478, 5522, 5664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159612928 dispatch_meta_bytes=229488 combine_payload_bytes=159612928 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159842416 ingress_bytes=235110400 link_bytes=235110400 token_bytes=319455344 weight_movement_bytes=75497472 total_nvlink_bytes=394952816 weight_to_token_ratio=0.236332 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788918489e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158916608 dispatch_meta_bytes=229792 combine_payload_bytes=158916608 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159146400 ingress_bytes=209248256 link_bytes=209248256 token_bytes=318063008 weight_movement_bytes=50331648 total_nvlink_bytes=368394656 weight_to_token_ratio=0.158244 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592137455e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38958 dup_rows=21034 dup_rows_remote=18396 dedup_ratio=0.320953 router_tokens_per_expert=[531, 510, 527, 502, 527, 472, 528, 519, 528, 541, 518, 523, 512, 491, 496, 473, 539, 498, 478, 562, 494, 508, 499, 522, 477, 540, 485, 520, 522, 501, 515, 494, 525, 525, 488, 493, 495, 507, 512, 523, 507, 504, 528, 507, 519, 501, 531, 513, 538, 542, 520, 524, 493, 485, 525, 527, 481, 496, 535, 538, 521, 488, 504, 521] payload_rows_per_dst=[5602, 5588, 5586, 5496, 5494, 5580, 5612, 5544] dup_rows_per_dst=[2492, 2714, 2614, 2586, 2654, 2640, 2696, 2638] combine_rows_per_src=[5602, 5588, 5586, 5496, 5494, 5580, 5612, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44612 payload_rows_remote=39036 dup_rows=20924 dup_rows_remote=18168 dedup_ratio=0.319275 router_tokens_per_expert=[527, 519, 517, 482, 489, 506, 517, 552, 518, 493, 525, 513, 519, 497, 493, 518, 483, 503, 515, 521, 489, 504, 505, 474, 527, 497, 525, 547, 525, 517, 523, 515, 515, 536, 521, 510, 511, 537, 476, 522, 517, 524, 543, 491, 479, 518, 523, 514, 493, 517, 543, 516, 472, 528, 525, 519, 504, 475, 514, 489, 501, 522, 533, 525] payload_rows_per_dst=[5576, 5578, 5662, 5526, 5486, 5574, 5724, 5486] dup_rows_per_dst=[2756, 2584, 2680, 2560, 2582, 2620, 2730, 2412] combine_rows_per_src=[5576, 5578, 5662, 5526, 5486, 5574, 5724, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158777344 dispatch_meta_bytes=229344 combine_payload_bytes=158777344 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159006688 ingress_bytes=234274816 link_bytes=234274816 token_bytes=317784032 weight_movement_bytes=75497472 total_nvlink_bytes=393281504 weight_to_token_ratio=0.237575 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782560660e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44446 payload_rows_remote=38768 dup_rows=21090 dup_rows_remote=18388 dedup_ratio=0.321808 router_tokens_per_expert=[531, 505, 502, 501, 481, 512, 493, 514, 479, 489, 489, 507, 501, 551, 521, 533, 520, 483, 491, 465, 516, 533, 522, 503, 506, 510, 527, 532, 527, 512, 527, 503, 534, 529, 520, 472, 479, 512, 511, 494, 533, 477, 521, 520, 517, 473, 545, 522, 503, 516, 567, 518, 529, 536, 516, 505, 512, 481, 536, 499, 550, 533, 504, 518] payload_rows_per_dst=[5526, 5520, 5514, 5578, 5526, 5540, 5678, 5564] dup_rows_per_dst=[2552, 2620, 2552, 2710, 2576, 2676, 2702, 2702] combine_rows_per_src=[5526, 5520, 5514, 5578, 5526, 5540, 5678, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=229784 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159637912 ingress_bytes=184573952 link_bytes=184573952 token_bytes=319046040 weight_movement_bytes=25165824 total_nvlink_bytes=344211864 weight_to_token_ratio=0.078878 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404394511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159776768 dispatch_meta_bytes=229848 combine_payload_bytes=159776768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160006616 ingress_bytes=184942592 link_bytes=184942592 token_bytes=319783384 weight_movement_bytes=25165824 total_nvlink_bytes=344949208 weight_to_token_ratio=0.078696 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407199436e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159571968 dispatch_meta_bytes=229416 combine_payload_bytes=159571968 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159801384 ingress_bytes=209903616 link_bytes=209903616 token_bytes=319373352 weight_movement_bytes=50331648 total_nvlink_bytes=369705000 weight_to_token_ratio=0.157595 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597123988e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159891456 dispatch_meta_bytes=228816 combine_payload_bytes=159891456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160120272 ingress_bytes=185057280 link_bytes=185057280 token_bytes=320011728 weight_movement_bytes=25165824 total_nvlink_bytes=345177552 weight_to_token_ratio=0.078640 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408072079e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158793728 dispatch_meta_bytes=228624 combine_payload_bytes=158793728 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159022352 ingress_bytes=234291200 link_bytes=234291200 token_bytes=317816080 weight_movement_bytes=75497472 total_nvlink_bytes=393313552 weight_to_token_ratio=0.237551 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782685323e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44578 payload_rows_remote=39052 dup_rows=20958 dup_rows_remote=18378 dedup_ratio=0.319794 router_tokens_per_expert=[552, 521, 540, 552, 506, 498, 512, 497, 537, 472, 518, 487, 497, 526, 522, 519, 527, 524, 501, 524, 516, 501, 515, 513, 523, 470, 484, 532, 516, 524, 546, 545, 502, 511, 498, 519, 519, 494, 518, 492, 505, 493, 531, 531, 490, 503, 520, 499, 516, 500, 528, 527, 512, 537, 447, 484, 524, 520, 498, 470, 483, 572, 501, 507] payload_rows_per_dst=[5668, 5548, 5630, 5692, 5526, 5512, 5464, 5538] dup_rows_per_dst=[2688, 2608, 2612, 2588, 2580, 2632, 2638, 2612] combine_rows_per_src=[5668, 5548, 5630, 5692, 5526, 5512, 5464, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38838 dup_rows=21142 dup_rows_remote=18604 dedup_ratio=0.322601 router_tokens_per_expert=[515, 514, 530, 481, 541, 530, 528, 525, 527, 509, 499, 497, 546, 489, 481, 527, 508, 500, 481, 512, 521, 489, 498, 490, 494, 477, 536, 516, 507, 532, 462, 523, 498, 534, 506, 539, 470, 482, 507, 494, 511, 545, 531, 509, 483, 531, 480, 498, 547, 487, 525, 529, 503, 508, 542, 505, 523, 529, 526, 500, 535, 533, 520, 553] payload_rows_per_dst=[5696, 5490, 5498, 5556, 5476, 5522, 5566, 5590] dup_rows_per_dst=[2632, 2660, 2500, 2538, 2584, 2654, 2726, 2848] combine_rows_per_src=[5696, 5490, 5498, 5556, 5476, 5522, 5566, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44480 payload_rows_remote=38978 dup_rows=21056 dup_rows_remote=18552 dedup_ratio=0.321289 router_tokens_per_expert=[506, 494, 492, 518, 540, 537, 488, 541, 515, 524, 524, 486, 504, 514, 512, 523, 544, 513, 532, 478, 495, 522, 503, 501, 514, 507, 537, 526, 496, 514, 499, 513, 509, 520, 545, 513, 544, 519, 511, 510, 468, 543, 505, 484, 505, 496, 488, 514, 517, 495, 513, 503, 522, 483, 513, 503, 522, 491, 485, 519, 538, 519, 541, 518] payload_rows_per_dst=[5628, 5560, 5514, 5568, 5598, 5502, 5536, 5574] dup_rows_per_dst=[2604, 2644, 2662, 2644, 2744, 2504, 2562, 2692] combine_rows_per_src=[5628, 5560, 5514, 5568, 5598, 5502, 5536, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44308 payload_rows_remote=38760 dup_rows=21228 dup_rows_remote=18612 dedup_ratio=0.323914 router_tokens_per_expert=[482, 498, 513, 516, 524, 491, 510, 507, 474, 542, 519, 502, 525, 515, 501, 547, 514, 499, 540, 482, 522, 528, 496, 501, 491, 528, 535, 507, 524, 486, 508, 518, 521, 545, 513, 517, 547, 533, 519, 514, 481, 497, 523, 527, 504, 505, 499, 487, 505, 456, 530, 503, 514, 562, 509, 497, 507, 518, 541, 514, 472, 505, 529, 529] payload_rows_per_dst=[5434, 5592, 5548, 5574, 5632, 5452, 5512, 5564] dup_rows_per_dst=[2648, 2658, 2616, 2620, 2786, 2594, 2640, 2666] combine_rows_per_src=[5434, 5592, 5548, 5574, 5632, 5452, 5512, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44382 payload_rows_remote=38854 dup_rows=21154 dup_rows_remote=18552 dedup_ratio=0.322784 router_tokens_per_expert=[493, 520, 542, 510, 495, 524, 469, 497, 495, 537, 506, 493, 503, 526, 512, 493, 508, 470, 539, 529, 526, 531, 536, 494, 526, 519, 494, 497, 540, 502, 543, 503, 508, 512, 527, 521, 541, 499, 526, 509, 540, 524, 521, 525, 454, 472, 506, 485, 520, 524, 510, 518, 501, 511, 531, 521, 509, 478, 545, 503, 501, 522, 499, 533] payload_rows_per_dst=[5462, 5528, 5626, 5542, 5590, 5532, 5556, 5546] dup_rows_per_dst=[2638, 2602, 2640, 2706, 2696, 2522, 2716, 2634] combine_rows_per_src=[5462, 5528, 5626, 5542, 5590, 5532, 5556, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159956992 dispatch_meta_bytes=229720 combine_payload_bytes=159956992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160186712 ingress_bytes=185122816 link_bytes=185122816 token_bytes=320143704 weight_movement_bytes=25165824 total_nvlink_bytes=345309528 weight_to_token_ratio=0.078608 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408570732e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159080448 dispatch_meta_bytes=229768 combine_payload_bytes=159080448 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159310216 ingress_bytes=234577920 link_bytes=234577920 token_bytes=318390664 weight_movement_bytes=75497472 total_nvlink_bytes=393888136 weight_to_token_ratio=0.237122 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784866931e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44280 payload_rows_remote=38742 dup_rows=21256 dup_rows_remote=18428 dedup_ratio=0.324341 router_tokens_per_expert=[508, 557, 533, 591, 502, 525, 511, 528, 520, 480, 547, 472, 518, 515, 492, 550, 495, 528, 510, 560, 502, 537, 500, 519, 531, 506, 480, 490, 513, 511, 515, 528, 521, 474, 537, 523, 520, 524, 504, 491, 472, 502, 500, 516, 507, 517, 513, 543, 508, 476, 476, 483, 486, 490, 507, 476, 535, 477, 495, 528, 532, 520, 543, 498] payload_rows_per_dst=[5482, 5604, 5598, 5438, 5432, 5610, 5578, 5538] dup_rows_per_dst=[2446, 2584, 2768, 2600, 2496, 2790, 2744, 2828] combine_rows_per_src=[5482, 5604, 5598, 5438, 5432, 5610, 5578, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=229488 combine_payload_bytes=158760960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158990448 ingress_bytes=183926784 link_bytes=183926784 token_bytes=317751408 weight_movement_bytes=25165824 total_nvlink_bytes=342917232 weight_to_token_ratio=0.079200 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399470311e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159653888 dispatch_meta_bytes=230120 combine_payload_bytes=159653888 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159884008 ingress_bytes=184819712 link_bytes=184819712 token_bytes=319537896 weight_movement_bytes=25165824 total_nvlink_bytes=344703720 weight_to_token_ratio=0.078757 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406264461e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44318 payload_rows_remote=38804 dup_rows=21218 dup_rows_remote=18466 dedup_ratio=0.323761 router_tokens_per_expert=[512, 523, 527, 498, 527, 505, 515, 502, 495, 509, 511, 514, 499, 516, 532, 526, 537, 498, 512, 493, 527, 478, 485, 508, 487, 539, 526, 477, 487, 554, 520, 478, 536, 511, 467, 468, 525, 503, 517, 510, 528, 522, 513, 544, 469, 545, 523, 532, 490, 557, 544, 522, 503, 528, 518, 503, 530, 500, 521, 479, 487, 551, 506, 499] payload_rows_per_dst=[5514, 5630, 5590, 5604, 5530, 5696, 5398, 5356] dup_rows_per_dst=[2752, 2574, 2834, 2534, 2544, 2978, 2534, 2468] combine_rows_per_src=[5514, 5630, 5590, 5604, 5530, 5696, 5398, 5356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44290 payload_rows_remote=38760 dup_rows=21246 dup_rows_remote=18526 dedup_ratio=0.324188 router_tokens_per_expert=[493, 501, 536, 527, 516, 505, 490, 490, 492, 537, 518, 494, 517, 505, 513, 511, 490, 511, 521, 531, 559, 544, 490, 533, 475, 524, 521, 530, 479, 507, 526, 487, 493, 499, 525, 517, 533, 535, 501, 491, 497, 506, 506, 519, 479, 509, 525, 503, 537, 545, 504, 469, 470, 504, 563, 533, 500, 531, 509, 512, 539, 510, 511, 520] payload_rows_per_dst=[5488, 5546, 5586, 5456, 5596, 5544, 5530, 5544] dup_rows_per_dst=[2628, 2628, 2772, 2642, 2592, 2544, 2720, 2720] combine_rows_per_src=[5488, 5546, 5586, 5456, 5596, 5544, 5530, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159145984 dispatch_meta_bytes=229624 combine_payload_bytes=159145984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159375608 ingress_bytes=184311808 link_bytes=184311808 token_bytes=318521592 weight_movement_bytes=25165824 total_nvlink_bytes=343687416 weight_to_token_ratio=0.079008 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402399898e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158687232 dispatch_meta_bytes=228680 combine_payload_bytes=158687232 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158915912 ingress_bytes=209018880 link_bytes=209018880 token_bytes=317603144 weight_movement_bytes=50331648 total_nvlink_bytes=367934792 weight_to_token_ratio=0.158473 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590392169e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=229080 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159170264 ingress_bytes=209272832 link_bytes=209272832 token_bytes=318111448 weight_movement_bytes=50331648 total_nvlink_bytes=368443096 weight_to_token_ratio=0.158220 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592324450e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=229144 combine_payload_bytes=158760960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158990104 ingress_bytes=183926784 link_bytes=183926784 token_bytes=317751064 weight_movement_bytes=25165824 total_nvlink_bytes=342916888 weight_to_token_ratio=0.079200 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399470311e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38974 dup_rows=21068 dup_rows_remote=18480 dedup_ratio=0.321472 router_tokens_per_expert=[507, 508, 519, 506, 512, 489, 507, 500, 508, 471, 518, 509, 512, 483, 499, 509, 477, 479, 554, 525, 520, 521, 551, 507, 527, 549, 497, 467, 548, 518, 499, 523, 506, 531, 473, 532, 498, 502, 510, 489, 503, 521, 521, 478, 522, 535, 491, 506, 472, 510, 505, 525, 527, 531, 519, 524, 468, 510, 542, 559, 548, 551, 514, 526] payload_rows_per_dst=[5500, 5488, 5594, 5650, 5494, 5554, 5608, 5580] dup_rows_per_dst=[2596, 2530, 2674, 2606, 2588, 2600, 2618, 2856] combine_rows_per_src=[5500, 5488, 5594, 5650, 5494, 5554, 5608, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44532 payload_rows_remote=39016 dup_rows=21004 dup_rows_remote=18394 dedup_ratio=0.320496 router_tokens_per_expert=[525, 508, 546, 499, 529, 539, 497, 494, 544, 545, 536, 500, 490, 516, 521, 497, 492, 500, 544, 511, 506, 476, 520, 510, 531, 506, 513, 473, 512, 514, 509, 505, 546, 534, 508, 530, 506, 497, 517, 510, 521, 505, 474, 552, 522, 534, 526, 501, 468, 497, 499, 530, 474, 521, 493, 506, 514, 522, 532, 553, 529, 503, 468, 468] payload_rows_per_dst=[5588, 5614, 5478, 5516, 5604, 5628, 5512, 5592] dup_rows_per_dst=[2686, 2684, 2640, 2610, 2692, 2642, 2464, 2586] combine_rows_per_src=[5588, 5614, 5478, 5516, 5604, 5628, 5512, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38906 dup_rows=21118 dup_rows_remote=18586 dedup_ratio=0.322235 router_tokens_per_expert=[539, 522, 549, 490, 515, 464, 520, 495, 469, 513, 506, 518, 488, 543, 507, 549, 517, 506, 486, 496, 504, 508, 491, 514, 534, 505, 539, 536, 522, 549, 497, 516, 513, 519, 483, 546, 497, 458, 552, 525, 534, 518, 528, 505, 517, 526, 500, 509, 504, 512, 508, 511, 521, 563, 507, 503, 514, 514, 469, 519, 498, 474, 528, 486] payload_rows_per_dst=[5554, 5534, 5512, 5612, 5548, 5548, 5618, 5492] dup_rows_per_dst=[2634, 2652, 2532, 2784, 2638, 2726, 2640, 2512] combine_rows_per_src=[5554, 5534, 5512, 5612, 5548, 5548, 5618, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44632 payload_rows_remote=39110 dup_rows=20904 dup_rows_remote=18322 dedup_ratio=0.318970 router_tokens_per_expert=[525, 485, 514, 537, 552, 476, 544, 523, 508, 515, 504, 518, 518, 523, 503, 512, 505, 511, 519, 488, 517, 449, 520, 509, 510, 491, 487, 547, 527, 503, 527, 484, 484, 548, 518, 503, 523, 519, 560, 520, 476, 509, 516, 491, 498, 517, 529, 516, 525, 510, 524, 469, 511, 516, 489, 512, 483, 522, 525, 532, 526, 528, 496, 522] payload_rows_per_dst=[5632, 5602, 5546, 5548, 5622, 5522, 5542, 5618] dup_rows_per_dst=[2680, 2600, 2490, 2604, 2728, 2582, 2570, 2650] combine_rows_per_src=[5632, 5602, 5546, 5548, 5622, 5522, 5542, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=39034 dup_rows=21014 dup_rows_remote=18370 dedup_ratio=0.320648 router_tokens_per_expert=[493, 506, 511, 505, 528, 522, 534, 510, 504, 486, 532, 484, 539, 500, 511, 510, 515, 522, 513, 497, 538, 494, 518, 492, 529, 524, 516, 566, 499, 487, 505, 484, 525, 517, 524, 540, 571, 514, 533, 512, 535, 522, 503, 454, 536, 510, 513, 486, 536, 481, 500, 485, 470, 557, 553, 483, 532, 514, 521, 496, 497, 511, 485, 478] payload_rows_per_dst=[5562, 5488, 5592, 5582, 5674, 5548, 5542, 5534] dup_rows_per_dst=[2656, 2644, 2586, 2638, 2798, 2570, 2588, 2534] combine_rows_per_src=[5562, 5488, 5592, 5582, 5674, 5548, 5542, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159637504 dispatch_meta_bytes=229816 combine_payload_bytes=159637504 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159867320 ingress_bytes=209969152 link_bytes=209969152 token_bytes=319504824 weight_movement_bytes=50331648 total_nvlink_bytes=369836472 weight_to_token_ratio=0.157530 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597622641e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159809536 dispatch_meta_bytes=229640 combine_payload_bytes=159809536 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160039176 ingress_bytes=235307008 link_bytes=235307008 token_bytes=319848712 weight_movement_bytes=75497472 total_nvlink_bytes=395346184 weight_to_token_ratio=0.236041 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790414449e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44456 payload_rows_remote=38790 dup_rows=21080 dup_rows_remote=18302 dedup_ratio=0.321655 router_tokens_per_expert=[491, 503, 535, 505, 479, 507, 483, 489, 518, 516, 519, 519, 540, 511, 528, 493, 507, 499, 534, 482, 485, 506, 553, 517, 497, 527, 529, 515, 508, 505, 517, 518, 512, 487, 542, 526, 502, 502, 493, 503, 523, 541, 523, 533, 523, 520, 520, 463, 502, 480, 498, 489, 554, 509, 480, 520, 550, 517, 502, 509, 552, 556, 497, 505] payload_rows_per_dst=[5472, 5440, 5550, 5520, 5358, 5656, 5794, 5666] dup_rows_per_dst=[2512, 2552, 2548, 2712, 2386, 2636, 2956, 2778] combine_rows_per_src=[5472, 5440, 5550, 5520, 5358, 5656, 5794, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38832 dup_rows=21088 dup_rows_remote=18300 dedup_ratio=0.321777 router_tokens_per_expert=[511, 525, 531, 540, 487, 527, 519, 556, 553, 525, 476, 484, 501, 500, 472, 468, 511, 527, 537, 529, 481, 508, 488, 503, 499, 547, 539, 517, 520, 524, 503, 481, 521, 494, 502, 522, 522, 477, 482, 498, 479, 535, 487, 550, 522, 521, 490, 480, 501, 546, 524, 530, 504, 481, 524, 521, 516, 542, 555, 493, 502, 498, 525, 535] payload_rows_per_dst=[5616, 5696, 5528, 5614, 5418, 5520, 5560, 5496] dup_rows_per_dst=[2788, 2642, 2640, 2636, 2616, 2608, 2702, 2456] combine_rows_per_src=[5616, 5696, 5528, 5614, 5418, 5520, 5560, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44456 payload_rows_remote=38944 dup_rows=21080 dup_rows_remote=18462 dedup_ratio=0.321655 router_tokens_per_expert=[483, 551, 489, 488, 508, 473, 497, 526, 479, 518, 532, 576, 493, 514, 513, 480, 584, 485, 538, 548, 555, 529, 535, 490, 518, 500, 493, 508, 537, 530, 515, 527, 537, 525, 490, 511, 545, 506, 499, 500, 489, 477, 482, 518, 529, 505, 510, 503, 519, 485, 516, 529, 500, 495, 506, 515, 524, 486, 493, 527, 534, 485, 524, 492] payload_rows_per_dst=[5518, 5558, 5666, 5606, 5556, 5470, 5512, 5570] dup_rows_per_dst=[2512, 2652, 2862, 2650, 2670, 2556, 2618, 2560] combine_rows_per_src=[5518, 5558, 5666, 5606, 5556, 5470, 5512, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229968 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159588944 ingress_bytes=184524800 link_bytes=184524800 token_bytes=318947920 weight_movement_bytes=25165824 total_nvlink_bytes=344113744 weight_to_token_ratio=0.078903 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404020521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160194560 dispatch_meta_bytes=229728 combine_payload_bytes=160194560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160424288 ingress_bytes=185360384 link_bytes=185360384 token_bytes=320618848 weight_movement_bytes=25165824 total_nvlink_bytes=345784672 weight_to_token_ratio=0.078491 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410378350e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=229616 combine_payload_bytes=159883264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160112880 ingress_bytes=185049088 link_bytes=185049088 token_bytes=319996144 weight_movement_bytes=25165824 total_nvlink_bytes=345161968 weight_to_token_ratio=0.078644 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408009747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158883840 dispatch_meta_bytes=228368 combine_payload_bytes=158883840 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159112208 ingress_bytes=184049664 link_bytes=184049664 token_bytes=317996048 weight_movement_bytes=25165824 total_nvlink_bytes=343161872 weight_to_token_ratio=0.079139 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400405285e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159055872 dispatch_meta_bytes=228528 combine_payload_bytes=159055872 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159284400 ingress_bytes=234553344 link_bytes=234553344 token_bytes=318340272 weight_movement_bytes=75497472 total_nvlink_bytes=393837744 weight_to_token_ratio=0.237160 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784679936e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159514624 dispatch_meta_bytes=229624 combine_payload_bytes=159514624 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159744248 ingress_bytes=184680448 link_bytes=184680448 token_bytes=319258872 weight_movement_bytes=25165824 total_nvlink_bytes=344424696 weight_to_token_ratio=0.078826 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405204823e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44520 payload_rows_remote=38894 dup_rows=21016 dup_rows_remote=18452 dedup_ratio=0.320679 router_tokens_per_expert=[490, 539, 525, 541, 547, 488, 466, 545, 527, 507, 526, 478, 485, 503, 518, 511, 542, 515, 488, 495, 513, 525, 473, 541, 500, 484, 482, 533, 530, 539, 502, 447, 482, 500, 526, 502, 513, 512, 547, 513, 532, 535, 522, 514, 493, 512, 510, 540, 505, 529, 548, 485, 507, 551, 468, 520, 507, 529, 568, 490, 518, 499, 515, 471] payload_rows_per_dst=[5610, 5512, 5502, 5496, 5626, 5602, 5578, 5594] dup_rows_per_dst=[2672, 2598, 2682, 2538, 2564, 2714, 2648, 2600] combine_rows_per_src=[5610, 5512, 5502, 5496, 5626, 5602, 5578, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44614 payload_rows_remote=39114 dup_rows=20922 dup_rows_remote=18256 dedup_ratio=0.319244 router_tokens_per_expert=[525, 514, 494, 476, 502, 527, 488, 521, 516, 512, 507, 499, 511, 522, 478, 508, 534, 531, 479, 580, 529, 475, 516, 507, 516, 502, 541, 521, 502, 486, 518, 497, 490, 510, 529, 559, 510, 517, 494, 503, 569, 498, 539, 510, 520, 501, 494, 473, 542, 519, 518, 508, 520, 524, 496, 499, 509, 493, 503, 535, 491, 512, 509, 540] payload_rows_per_dst=[5524, 5546, 5632, 5500, 5590, 5600, 5630, 5592] dup_rows_per_dst=[2570, 2560, 2670, 2666, 2634, 2608, 2622, 2592] combine_rows_per_src=[5524, 5546, 5632, 5500, 5590, 5600, 5630, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44654 payload_rows_remote=39058 dup_rows=20882 dup_rows_remote=18300 dedup_ratio=0.318634 router_tokens_per_expert=[518, 495, 493, 514, 535, 482, 514, 520, 528, 528, 510, 519, 496, 534, 513, 511, 538, 518, 495, 483, 490, 541, 521, 503, 510, 532, 521, 531, 506, 509, 516, 500, 529, 446, 549, 504, 538, 511, 503, 531, 533, 501, 512, 552, 506, 503, 513, 491, 514, 497, 497, 484, 504, 485, 465, 514, 526, 535, 557, 503, 555, 510, 469, 507] payload_rows_per_dst=[5502, 5640, 5596, 5608, 5602, 5618, 5464, 5624] dup_rows_per_dst=[2640, 2638, 2582, 2642, 2620, 2604, 2456, 2700] combine_rows_per_src=[5502, 5640, 5596, 5608, 5602, 5618, 5464, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44406 payload_rows_remote=38890 dup_rows=21130 dup_rows_remote=18468 dedup_ratio=0.322418 router_tokens_per_expert=[505, 523, 511, 503, 525, 518, 496, 536, 532, 496, 471, 494, 518, 519, 539, 531, 565, 480, 501, 504, 518, 509, 514, 507, 520, 509, 498, 484, 519, 535, 515, 532, 531, 506, 532, 505, 520, 537, 480, 500, 512, 546, 531, 514, 482, 518, 474, 512, 508, 514, 496, 537, 496, 509, 509, 526, 522, 489, 486, 508, 515, 488, 523, 515] payload_rows_per_dst=[5600, 5548, 5522, 5516, 5598, 5516, 5606, 5500] dup_rows_per_dst=[2634, 2652, 2674, 2708, 2624, 2662, 2584, 2592] combine_rows_per_src=[5600, 5548, 5522, 5516, 5598, 5516, 5606, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=229384 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159539208 ingress_bytes=209641472 link_bytes=209641472 token_bytes=318849032 weight_movement_bytes=50331648 total_nvlink_bytes=369180680 weight_to_token_ratio=0.157854 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595129375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44596 payload_rows_remote=39020 dup_rows=20940 dup_rows_remote=18318 dedup_ratio=0.319519 router_tokens_per_expert=[573, 515, 512, 525, 521, 524, 518, 530, 519, 492, 481, 481, 540, 515, 534, 537, 504, 513, 515, 501, 503, 507, 478, 518, 523, 476, 492, 509, 528, 493, 529, 496, 537, 482, 507, 544, 545, 525, 510, 520, 533, 497, 523, 528, 534, 513, 500, 491, 462, 510, 538, 507, 488, 481, 507, 488, 513, 536, 513, 498, 480, 507, 552, 497] payload_rows_per_dst=[5740, 5576, 5492, 5490, 5552, 5634, 5504, 5608] dup_rows_per_dst=[2696, 2622, 2586, 2602, 2788, 2604, 2458, 2584] combine_rows_per_src=[5740, 5576, 5492, 5490, 5552, 5634, 5504, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160210944 dispatch_meta_bytes=229480 combine_payload_bytes=160210944 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160440424 ingress_bytes=185376768 link_bytes=185376768 token_bytes=320651368 weight_movement_bytes=25165824 total_nvlink_bytes=345817192 weight_to_token_ratio=0.078483 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410503013e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38764 dup_rows=21114 dup_rows_remote=18196 dedup_ratio=0.322174 router_tokens_per_expert=[537, 540, 508, 488, 521, 501, 492, 512, 512, 507, 535, 533, 525, 476, 525, 517, 496, 514, 521, 505, 472, 515, 510, 508, 546, 505, 495, 530, 486, 520, 510, 528, 518, 513, 520, 532, 501, 509, 523, 491, 494, 539, 505, 532, 521, 514, 502, 518, 515, 511, 517, 526, 506, 500, 486, 513, 527, 486, 492, 471, 574, 512, 478, 532] payload_rows_per_dst=[5572, 5358, 5772, 5410, 5530, 5534, 5588, 5658] dup_rows_per_dst=[2626, 2354, 2858, 2398, 2684, 2618, 2658, 2918] combine_rows_per_src=[5572, 5358, 5772, 5410, 5530, 5534, 5588, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44588 payload_rows_remote=39012 dup_rows=20948 dup_rows_remote=18190 dedup_ratio=0.319641 router_tokens_per_expert=[506, 514, 485, 523, 513, 510, 526, 500, 529, 514, 537, 532, 481, 518, 554, 520, 517, 498, 515, 464, 505, 465, 529, 521, 514, 520, 523, 517, 498, 536, 504, 525, 514, 512, 521, 510, 519, 537, 540, 531, 522, 511, 532, 515, 523, 511, 520, 496, 500, 543, 498, 499, 490, 494, 513, 516, 497, 501, 499, 501, 480, 530, 473, 507] payload_rows_per_dst=[5576, 5662, 5480, 5622, 5670, 5544, 5560, 5474] dup_rows_per_dst=[2758, 2780, 2492, 2660, 2698, 2512, 2546, 2502] combine_rows_per_src=[5576, 5662, 5480, 5622, 5670, 5544, 5560, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38924 dup_rows=21076 dup_rows_remote=18474 dedup_ratio=0.321594 router_tokens_per_expert=[477, 513, 496, 518, 514, 520, 483, 511, 508, 500, 529, 531, 543, 542, 517, 504, 524, 521, 470, 530, 555, 490, 521, 461, 519, 521, 506, 509, 513, 513, 539, 475, 508, 517, 492, 507, 488, 490, 563, 542, 567, 498, 537, 516, 524, 524, 520, 531, 494, 518, 474, 524, 507, 539, 501, 512, 543, 456, 509, 504, 492, 494, 506, 498] payload_rows_per_dst=[5514, 5640, 5530, 5546, 5556, 5668, 5536, 5470] dup_rows_per_dst=[2550, 2708, 2614, 2644, 2658, 2766, 2602, 2534] combine_rows_per_src=[5514, 5640, 5530, 5546, 5556, 5668, 5536, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159981568 dispatch_meta_bytes=229432 combine_payload_bytes=159981568 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160211000 ingress_bytes=235479040 link_bytes=235479040 token_bytes=320192568 weight_movement_bytes=75497472 total_nvlink_bytes=395690040 weight_to_token_ratio=0.235788 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791723413e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=229432 combine_payload_bytes=159293440 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159522872 ingress_bytes=234790912 link_bytes=234790912 token_bytes=318816312 weight_movement_bytes=75497472 total_nvlink_bytes=394313784 weight_to_token_ratio=0.236806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786487554e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159825920 dispatch_meta_bytes=229352 combine_payload_bytes=159825920 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160055272 ingress_bytes=210157568 link_bytes=210157568 token_bytes=319881192 weight_movement_bytes=50331648 total_nvlink_bytes=370212840 weight_to_token_ratio=0.157345 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599056269e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158777344 dispatch_meta_bytes=227840 combine_payload_bytes=158777344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159005184 ingress_bytes=183943168 link_bytes=183943168 token_bytes=317782528 weight_movement_bytes=25165824 total_nvlink_bytes=342948352 weight_to_token_ratio=0.079192 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399594974e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159793152 dispatch_meta_bytes=228808 combine_payload_bytes=159793152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160021960 ingress_bytes=184958976 link_bytes=184958976 token_bytes=319815112 weight_movement_bytes=25165824 total_nvlink_bytes=344980936 weight_to_token_ratio=0.078689 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407324099e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=229592 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159662296 ingress_bytes=209764352 link_bytes=209764352 token_bytes=319095000 weight_movement_bytes=50331648 total_nvlink_bytes=369426648 weight_to_token_ratio=0.157732 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596064349e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38902 dup_rows=21124 dup_rows_remote=18524 dedup_ratio=0.322327 router_tokens_per_expert=[528, 504, 528, 509, 482, 520, 489, 524, 503, 567, 499, 460, 517, 498, 529, 486, 511, 510, 506, 524, 555, 545, 487, 527, 496, 511, 473, 502, 532, 506, 512, 512, 482, 501, 519, 506, 529, 497, 517, 504, 498, 527, 492, 507, 553, 544, 529, 509, 500, 514, 518, 505, 486, 531, 522, 531, 513, 528, 519, 530, 509, 517, 480, 499] payload_rows_per_dst=[5568, 5544, 5606, 5492, 5510, 5638, 5536, 5518] dup_rows_per_dst=[2600, 2574, 2724, 2596, 2600, 2680, 2678, 2672] combine_rows_per_src=[5568, 5544, 5606, 5492, 5510, 5638, 5536, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=39000 dup_rows=20968 dup_rows_remote=18342 dedup_ratio=0.319946 router_tokens_per_expert=[538, 486, 489, 539, 493, 547, 532, 508, 518, 524, 515, 540, 531, 530, 528, 497, 529, 496, 522, 503, 480, 475, 524, 485, 526, 527, 487, 539, 491, 470, 539, 518, 523, 497, 508, 506, 520, 486, 495, 496, 542, 485, 504, 467, 494, 495, 536, 495, 564, 529, 548, 527, 500, 548, 503, 513, 494, 474, 489, 547, 510, 541, 505, 501] payload_rows_per_dst=[5668, 5668, 5450, 5568, 5510, 5510, 5700, 5494] dup_rows_per_dst=[2596, 2698, 2578, 2626, 2552, 2526, 2764, 2628] combine_rows_per_src=[5668, 5668, 5450, 5568, 5510, 5510, 5700, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44296 payload_rows_remote=38792 dup_rows=21240 dup_rows_remote=18714 dedup_ratio=0.324097 router_tokens_per_expert=[508, 532, 464, 515, 485, 509, 528, 518, 512, 528, 537, 567, 534, 523, 521, 487, 524, 515, 521, 501, 475, 488, 478, 513, 513, 511, 505, 516, 478, 550, 517, 534, 487, 484, 510, 515, 514, 517, 513, 484, 537, 514, 493, 503, 528, 478, 537, 553, 469, 539, 532, 496, 529, 517, 532, 522, 499, 534, 540, 514, 454, 473, 519, 525] payload_rows_per_dst=[5444, 5706, 5504, 5500, 5470, 5514, 5578, 5580] dup_rows_per_dst=[2674, 2712, 2526, 2748, 2578, 2772, 2694, 2536] combine_rows_per_src=[5444, 5706, 5504, 5500, 5470, 5514, 5578, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=229704 combine_payload_bytes=159342592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159572296 ingress_bytes=184508416 link_bytes=184508416 token_bytes=318914888 weight_movement_bytes=25165824 total_nvlink_bytes=344080712 weight_to_token_ratio=0.078911 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403895858e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44556 payload_rows_remote=39010 dup_rows=20980 dup_rows_remote=18476 dedup_ratio=0.320129 router_tokens_per_expert=[542, 489, 535, 515, 485, 490, 484, 473, 544, 505, 489, 497, 503, 480, 535, 517, 534, 508, 516, 491, 515, 536, 508, 527, 531, 487, 530, 499, 500, 547, 503, 533, 566, 475, 478, 539, 526, 497, 515, 499, 506, 502, 507, 503, 499, 500, 515, 493, 534, 530, 532, 465, 530, 524, 509, 504, 527, 542, 528, 512, 505, 526, 495, 537] payload_rows_per_dst=[5556, 5542, 5546, 5604, 5552, 5546, 5572, 5638] dup_rows_per_dst=[2470, 2598, 2724, 2656, 2638, 2504, 2684, 2706] combine_rows_per_src=[5556, 5542, 5546, 5604, 5552, 5546, 5572, 5638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159744000 dispatch_meta_bytes=229368 combine_payload_bytes=159744000 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159973368 ingress_bytes=235241472 link_bytes=235241472 token_bytes=319717368 weight_movement_bytes=75497472 total_nvlink_bytes=395214840 weight_to_token_ratio=0.236138 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789915795e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38962 dup_rows=21036 dup_rows_remote=18460 dedup_ratio=0.320984 router_tokens_per_expert=[518, 529, 509, 461, 507, 504, 486, 469, 515, 553, 491, 524, 498, 496, 490, 490, 505, 548, 499, 500, 496, 496, 528, 482, 558, 513, 503, 514, 546, 539, 523, 521, 551, 490, 526, 526, 501, 548, 530, 515, 489, 506, 559, 543, 519, 508, 483, 490, 509, 493, 523, 506, 514, 519, 529, 510, 517, 505, 510, 485, 501, 514, 548, 490] payload_rows_per_dst=[5518, 5538, 5508, 5624, 5692, 5562, 5554, 5504] dup_rows_per_dst=[2448, 2576, 2600, 2810, 2682, 2632, 2652, 2636] combine_rows_per_src=[5518, 5538, 5508, 5624, 5692, 5562, 5554, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44376 payload_rows_remote=38758 dup_rows=21160 dup_rows_remote=18318 dedup_ratio=0.322876 router_tokens_per_expert=[495, 524, 499, 528, 530, 496, 509, 489, 524, 510, 514, 499, 484, 534, 517, 486, 489, 514, 556, 497, 521, 513, 531, 511, 499, 533, 510, 499, 523, 521, 512, 521, 493, 501, 521, 505, 521, 529, 518, 518, 520, 530, 533, 504, 483, 504, 525, 493, 540, 529, 498, 542, 488, 473, 516, 513, 492, 497, 540, 522, 514, 502, 515, 501] payload_rows_per_dst=[5486, 5464, 5556, 5434, 5588, 5558, 5672, 5618] dup_rows_per_dst=[2398, 2672, 2708, 2490, 2642, 2626, 2782, 2842] combine_rows_per_src=[5486, 5464, 5556, 5434, 5588, 5558, 5672, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38782 dup_rows=21000 dup_rows_remote=18022 dedup_ratio=0.320435 router_tokens_per_expert=[512, 535, 538, 548, 519, 519, 503, 486, 501, 475, 552, 506, 505, 539, 519, 532, 481, 498, 491, 507, 521, 506, 481, 513, 525, 474, 527, 530, 520, 543, 547, 517, 499, 489, 533, 547, 508, 528, 515, 524, 494, 465, 497, 531, 483, 532, 521, 534, 488, 518, 496, 523, 532, 495, 514, 510, 498, 498, 500, 485, 531, 500, 513, 497] payload_rows_per_dst=[5754, 5584, 5466, 5656, 5566, 5554, 5388, 5568] dup_rows_per_dst=[2978, 2736, 2530, 2658, 2758, 2512, 2352, 2476] combine_rows_per_src=[5754, 5584, 5466, 5656, 5566, 5554, 5388, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44504 payload_rows_remote=39092 dup_rows=21032 dup_rows_remote=18476 dedup_ratio=0.320923 router_tokens_per_expert=[507, 479, 530, 513, 494, 503, 496, 515, 498, 497, 532, 550, 500, 501, 514, 540, 527, 493, 477, 541, 496, 501, 509, 475, 537, 486, 511, 552, 500, 493, 514, 502, 530, 526, 525, 505, 538, 559, 560, 516, 553, 512, 481, 489, 537, 528, 556, 517, 464, 495, 491, 547, 471, 497, 522, 497, 522, 494, 528, 486, 484, 533, 522, 500] payload_rows_per_dst=[5558, 5550, 5548, 5552, 5740, 5582, 5412, 5562] dup_rows_per_dst=[2516, 2714, 2490, 2638, 2778, 2764, 2556, 2576] combine_rows_per_src=[5558, 5550, 5548, 5552, 5740, 5582, 5412, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158892032 dispatch_meta_bytes=230024 combine_payload_bytes=158892032 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159122056 ingress_bytes=184057856 link_bytes=184057856 token_bytes=318014088 weight_movement_bytes=25165824 total_nvlink_bytes=343179912 weight_to_token_ratio=0.079134 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400467617e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=229944 combine_payload_bytes=159784960 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160014904 ingress_bytes=235282432 link_bytes=235282432 token_bytes=319799864 weight_movement_bytes=75497472 total_nvlink_bytes=395297336 weight_to_token_ratio=0.236077 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790227454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159588352 dispatch_meta_bytes=229688 combine_payload_bytes=159588352 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159818040 ingress_bytes=209920000 link_bytes=209920000 token_bytes=319406392 weight_movement_bytes=50331648 total_nvlink_bytes=369738040 weight_to_token_ratio=0.157579 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597248651e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158752768 dispatch_meta_bytes=228304 combine_payload_bytes=158752768 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158981072 ingress_bytes=183918592 link_bytes=183918592 token_bytes=317733840 weight_movement_bytes=25165824 total_nvlink_bytes=342899664 weight_to_token_ratio=0.079204 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399407979e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158851072 dispatch_meta_bytes=227216 combine_payload_bytes=158851072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159078288 ingress_bytes=184016896 link_bytes=184016896 token_bytes=317929360 weight_movement_bytes=25165824 total_nvlink_bytes=343095184 weight_to_token_ratio=0.079155 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400155959e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44598 payload_rows_remote=38994 dup_rows=20938 dup_rows_remote=18244 dedup_ratio=0.319489 router_tokens_per_expert=[507, 507, 517, 538, 521, 465, 523, 515, 513, 520, 509, 484, 516, 517, 534, 527, 536, 541, 520, 507, 504, 528, 514, 514, 546, 511, 489, 527, 552, 475, 510, 488, 495, 517, 512, 511, 573, 501, 544, 496, 514, 480, 510, 512, 550, 512, 503, 537, 493, 542, 525, 489, 462, 508, 498, 493, 483, 542, 487, 494, 491, 512, 508, 499] payload_rows_per_dst=[5618, 5514, 5618, 5552, 5604, 5578, 5568, 5546] dup_rows_per_dst=[2568, 2726, 2710, 2644, 2694, 2658, 2452, 2486] combine_rows_per_src=[5618, 5514, 5618, 5552, 5604, 5578, 5568, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160120832 dispatch_meta_bytes=230272 combine_payload_bytes=160120832 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160351104 ingress_bytes=185286656 link_bytes=185286656 token_bytes=320471936 weight_movement_bytes=25165824 total_nvlink_bytes=345637760 weight_to_token_ratio=0.078527 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409817365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=39034 dup_rows=21014 dup_rows_remote=18492 dedup_ratio=0.320648 router_tokens_per_expert=[516, 502, 506, 559, 564, 532, 504, 529, 505, 553, 492, 467, 498, 520, 483, 531, 524, 534, 530, 540, 521, 488, 496, 507, 505, 496, 484, 500, 513, 503, 504, 500, 535, 502, 486, 492, 495, 506, 521, 498, 524, 507, 555, 491, 542, 516, 493, 500, 481, 489, 471, 516, 502, 525, 503, 535, 504, 490, 539, 530, 498, 527, 562, 527] payload_rows_per_dst=[5682, 5618, 5602, 5488, 5506, 5566, 5500, 5560] dup_rows_per_dst=[2742, 2480, 2678, 2522, 2564, 2690, 2544, 2794] combine_rows_per_src=[5682, 5618, 5602, 5488, 5506, 5566, 5500, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159719424 dispatch_meta_bytes=228952 combine_payload_bytes=159719424 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159948376 ingress_bytes=184885248 link_bytes=184885248 token_bytes=319667800 weight_movement_bytes=25165824 total_nvlink_bytes=344833624 weight_to_token_ratio=0.078725 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406763114e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=230104 combine_payload_bytes=159883264 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160113368 ingress_bytes=235380736 link_bytes=235380736 token_bytes=319996632 weight_movement_bytes=75497472 total_nvlink_bytes=395494104 weight_to_token_ratio=0.235932 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790975434e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44556 payload_rows_remote=38956 dup_rows=20980 dup_rows_remote=18520 dedup_ratio=0.320129 router_tokens_per_expert=[546, 534, 495, 506, 525, 540, 532, 508, 490, 522, 551, 533, 489, 548, 547, 481, 518, 483, 503, 474, 510, 526, 486, 530, 519, 535, 497, 480, 560, 532, 524, 538, 499, 507, 504, 513, 508, 489, 514, 511, 505, 493, 478, 452, 533, 500, 520, 518, 532, 514, 463, 549, 544, 529, 505, 511, 520, 553, 520, 487, 507, 505, 475, 448] payload_rows_per_dst=[5592, 5624, 5600, 5678, 5518, 5476, 5552, 5516] dup_rows_per_dst=[2780, 2698, 2460, 2692, 2572, 2522, 2742, 2514] combine_rows_per_src=[5592, 5624, 5600, 5678, 5518, 5476, 5552, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44692 payload_rows_remote=39138 dup_rows=20844 dup_rows_remote=18172 dedup_ratio=0.318054 router_tokens_per_expert=[499, 540, 527, 521, 523, 509, 487, 549, 503, 500, 519, 494, 500, 503, 513, 507, 524, 479, 589, 489, 478, 510, 518, 470, 526, 502, 515, 522, 530, 519, 487, 525, 509, 514, 531, 520, 454, 470, 529, 470, 499, 511, 519, 526, 505, 569, 491, 493, 531, 525, 523, 514, 534, 514, 537, 528, 543, 501, 512, 534, 499, 493, 478, 515] payload_rows_per_dst=[5646, 5546, 5530, 5662, 5482, 5554, 5698, 5574] dup_rows_per_dst=[2664, 2532, 2584, 2590, 2512, 2672, 2714, 2576] combine_rows_per_src=[5646, 5546, 5530, 5662, 5482, 5554, 5698, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44582 payload_rows_remote=39102 dup_rows=20954 dup_rows_remote=18404 dedup_ratio=0.319733 router_tokens_per_expert=[480, 516, 482, 529, 509, 545, 513, 534, 512, 444, 542, 505, 510, 489, 519, 494, 467, 498, 534, 512, 485, 531, 484, 518, 554, 533, 482, 516, 561, 519, 518, 541, 526, 521, 480, 529, 504, 509, 532, 517, 512, 554, 513, 510, 510, 516, 505, 509, 528, 495, 558, 488, 508, 516, 499, 503, 526, 516, 520, 505, 504, 504, 484, 491] payload_rows_per_dst=[5604, 5480, 5536, 5704, 5550, 5570, 5564, 5574] dup_rows_per_dst=[2612, 2550, 2522, 2744, 2686, 2688, 2626, 2526] combine_rows_per_src=[5604, 5480, 5536, 5704, 5550, 5570, 5564, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44316 payload_rows_remote=38800 dup_rows=21220 dup_rows_remote=18508 dedup_ratio=0.323792 router_tokens_per_expert=[507, 508, 498, 486, 499, 531, 514, 505, 500, 533, 566, 477, 489, 499, 528, 519, 546, 524, 501, 511, 535, 522, 525, 480, 522, 516, 475, 520, 513, 534, 523, 523, 465, 533, 502, 549, 547, 513, 487, 498, 496, 493, 539, 492, 552, 504, 501, 524, 488, 487, 527, 501, 527, 500, 543, 503, 543, 488, 486, 487, 520, 458, 545, 541] payload_rows_per_dst=[5248, 5554, 5566, 5554, 5582, 5558, 5738, 5516] dup_rows_per_dst=[2182, 2640, 2630, 2698, 2634, 2644, 3080, 2712] combine_rows_per_src=[5248, 5554, 5566, 5554, 5582, 5558, 5738, 5516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44700 payload_rows_remote=39160 dup_rows=20836 dup_rows_remote=18368 dedup_ratio=0.317932 router_tokens_per_expert=[525, 532, 509, 478, 542, 530, 515, 525, 538, 458, 520, 540, 494, 539, 498, 546, 480, 496, 491, 509, 473, 502, 502, 510, 511, 515, 499, 563, 536, 508, 520, 536, 484, 498, 477, 520, 507, 505, 498, 492, 537, 521, 522, 507, 555, 490, 502, 522, 531, 496, 491, 519, 504, 478, 478, 507, 513, 502, 516, 546, 502, 539, 552, 517] payload_rows_per_dst=[5558, 5630, 5438, 5714, 5492, 5626, 5540, 5702] dup_rows_per_dst=[2754, 2636, 2488, 2662, 2470, 2686, 2468, 2672] combine_rows_per_src=[5558, 5630, 5438, 5714, 5492, 5626, 5540, 5702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38904 dup_rows=21114 dup_rows_remote=18562 dedup_ratio=0.322174 router_tokens_per_expert=[485, 538, 502, 511, 514, 461, 506, 502, 533, 560, 504, 520, 506, 496, 506, 510, 521, 508, 446, 489, 546, 499, 501, 532, 501, 517, 495, 515, 475, 541, 509, 536, 532, 504, 498, 486, 518, 505, 533, 474, 518, 557, 560, 515, 524, 496, 533, 517, 522, 458, 493, 513, 519, 531, 507, 534, 500, 517, 539, 499, 545, 531, 519, 486] payload_rows_per_dst=[5518, 5688, 5568, 5540, 5510, 5468, 5556, 5574] dup_rows_per_dst=[2552, 2680, 2826, 2606, 2590, 2564, 2598, 2698] combine_rows_per_src=[5518, 5688, 5568, 5540, 5510, 5468, 5556, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=229904 combine_payload_bytes=159563776 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159793680 ingress_bytes=184729600 link_bytes=184729600 token_bytes=319357456 weight_movement_bytes=25165824 total_nvlink_bytes=344523280 weight_to_token_ratio=0.078801 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405578813e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160161792 dispatch_meta_bytes=230024 combine_payload_bytes=160161792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160391816 ingress_bytes=185327616 link_bytes=185327616 token_bytes=320553608 weight_movement_bytes=25165824 total_nvlink_bytes=345719432 weight_to_token_ratio=0.078507 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410129024e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160309248 dispatch_meta_bytes=229240 combine_payload_bytes=160309248 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160538488 ingress_bytes=210640896 link_bytes=210640896 token_bytes=320847736 weight_movement_bytes=50331648 total_nvlink_bytes=371179384 weight_to_token_ratio=0.156871 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602733836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158924800 dispatch_meta_bytes=229232 combine_payload_bytes=158924800 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159154032 ingress_bytes=184090624 link_bytes=184090624 token_bytes=318078832 weight_movement_bytes=25165824 total_nvlink_bytes=343244656 weight_to_token_ratio=0.079118 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400716944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160399360 dispatch_meta_bytes=230112 combine_payload_bytes=160399360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160629472 ingress_bytes=185565184 link_bytes=185565184 token_bytes=321028832 weight_movement_bytes=25165824 total_nvlink_bytes=346194656 weight_to_token_ratio=0.078391 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411936642e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38828 dup_rows=21114 dup_rows_remote=18362 dedup_ratio=0.322174 router_tokens_per_expert=[514, 513, 497, 521, 526, 552, 523, 498, 522, 483, 505, 521, 552, 500, 522, 471, 507, 512, 486, 500, 523, 558, 518, 494, 534, 509, 481, 502, 495, 521, 499, 531, 537, 525, 511, 501, 538, 520, 527, 514, 497, 476, 473, 523, 505, 542, 518, 507, 510, 506, 515, 513, 494, 516, 508, 528, 518, 515, 503, 502, 502, 503, 513, 518] payload_rows_per_dst=[5580, 5478, 5572, 5560, 5594, 5548, 5544, 5546] dup_rows_per_dst=[2708, 2674, 2624, 2584, 2752, 2534, 2636, 2602] combine_rows_per_src=[5580, 5478, 5572, 5560, 5594, 5548, 5544, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44404 payload_rows_remote=38840 dup_rows=21132 dup_rows_remote=18438 dedup_ratio=0.322449 router_tokens_per_expert=[506, 523, 533, 499, 533, 530, 475, 543, 496, 553, 515, 476, 517, 505, 568, 537, 516, 524, 510, 498, 521, 513, 510, 540, 514, 527, 525, 508, 552, 514, 481, 508, 533, 475, 508, 506, 509, 513, 506, 525, 492, 533, 521, 524, 509, 546, 532, 493, 483, 515, 502, 463, 503, 527, 496, 497, 485, 474, 489, 488, 549, 505, 500, 497] payload_rows_per_dst=[5592, 5596, 5606, 5564, 5582, 5558, 5458, 5448] dup_rows_per_dst=[2692, 2738, 2658, 2694, 2568, 2742, 2514, 2526] combine_rows_per_src=[5592, 5596, 5606, 5564, 5582, 5558, 5458, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229864 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159580648 ingress_bytes=209682432 link_bytes=209682432 token_bytes=318931432 weight_movement_bytes=50331648 total_nvlink_bytes=369263080 weight_to_token_ratio=0.157813 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595441033e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5090430288 prefetch_bytes=377487360 reduce_bytes=276824064 weight_movement_bytes=654311424 egress_bytes=2547043664 ingress_bytes=3197698048 link_bytes=3197698048 total_nvlink_bytes=5744741712 weight_to_token_ratio=0.128538 prefetch_share_of_ingress=0.118050 dedup_ratio=0.322031 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=11 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.520674213e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5105795592 prefetch_bytes=301989888 reduce_bytes=452984832 weight_movement_bytes=754974720 egress_bytes=2554733064 ingress_bytes=3306037248 link_bytes=3306037248 total_nvlink_bytes=5860770312 weight_to_token_ratio=0.147866 prefetch_share_of_ingress=0.091345 dedup_ratio=0.321331 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=18 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=19812 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.572195221e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=229112 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159317752 ingress_bytes=209420288 link_bytes=209420288 token_bytes=318406392 weight_movement_bytes=50331648 total_nvlink_bytes=368738040 weight_to_token_ratio=0.158074 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593446420e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159039488 dispatch_meta_bytes=228760 combine_payload_bytes=159039488 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159268248 ingress_bytes=209371136 link_bytes=209371136 token_bytes=318307736 weight_movement_bytes=50331648 total_nvlink_bytes=368639384 weight_to_token_ratio=0.158123 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593072430e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44426 payload_rows_remote=38878 dup_rows=21110 dup_rows_remote=18434 dedup_ratio=0.322113 router_tokens_per_expert=[508, 543, 532, 498, 555, 521, 532, 508, 503, 479, 517, 511, 490, 508, 491, 520, 541, 524, 514, 509, 509, 493, 510, 512, 510, 513, 526, 524, 507, 537, 517, 485, 526, 542, 520, 504, 522, 483, 482, 500, 497, 491, 528, 528, 454, 512, 514, 509, 521, 473, 510, 495, 517, 528, 504, 496, 501, 569, 532, 524, 507, 472, 539, 521] payload_rows_per_dst=[5616, 5490, 5548, 5554, 5490, 5566, 5542, 5620] dup_rows_per_dst=[2778, 2548, 2676, 2684, 2668, 2500, 2546, 2710] combine_rows_per_src=[5616, 5490, 5548, 5554, 5490, 5566, 5542, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44628 payload_rows_remote=39148 dup_rows=20908 dup_rows_remote=18282 dedup_ratio=0.319031 router_tokens_per_expert=[512, 537, 548, 507, 519, 539, 494, 499, 490, 529, 512, 527, 512, 487, 492, 504, 509, 559, 486, 531, 510, 508, 509, 524, 514, 511, 531, 503, 554, 504, 489, 502, 497, 502, 547, 524, 530, 536, 511, 528, 479, 476, 522, 488, 510, 496, 548, 497, 484, 524, 528, 487, 509, 528, 500, 502, 544, 500, 506, 493, 490, 521, 483, 526] payload_rows_per_dst=[5662, 5480, 5620, 5572, 5600, 5558, 5544, 5592] dup_rows_per_dst=[2648, 2626, 2652, 2644, 2750, 2474, 2580, 2534] combine_rows_per_src=[5662, 5480, 5620, 5572, 5600, 5558, 5544, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44792 payload_rows_remote=39210 dup_rows=20744 dup_rows_remote=18060 dedup_ratio=0.316528 router_tokens_per_expert=[484, 509, 473, 525, 507, 521, 525, 513, 502, 533, 538, 522, 522, 464, 521, 459, 498, 517, 498, 534, 514, 498, 521, 486, 527, 520, 506, 489, 517, 505, 515, 499, 510, 493, 493, 513, 554, 525, 533, 510, 535, 531, 522, 519, 522, 506, 495, 503, 504, 515, 516, 508, 519, 515, 523, 519, 521, 543, 477, 523, 480, 520, 531, 528] payload_rows_per_dst=[5548, 5524, 5584, 5630, 5638, 5582, 5628, 5658] dup_rows_per_dst=[2566, 2598, 2548, 2526, 2624, 2684, 2610, 2588] combine_rows_per_src=[5548, 5524, 5584, 5630, 5638, 5582, 5628, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5111628992 prefetch_bytes=377487360 reduce_bytes=427819008 weight_movement_bytes=805306368 egress_bytes=2557650112 ingress_bytes=3359285248 link_bytes=3359285248 total_nvlink_bytes=5916935360 weight_to_token_ratio=0.157544 prefetch_share_of_ingress=0.112371 dedup_ratio=0.320547 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=17 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.039292 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.597517456e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5100864280 prefetch_bytes=377487360 reduce_bytes=276824064 weight_movement_bytes=654311424 egress_bytes=2552267544 ingress_bytes=3202908160 link_bytes=3202908160 total_nvlink_bytes=5755175704 weight_to_token_ratio=0.128275 prefetch_share_of_ingress=0.117858 dedup_ratio=0.321791 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=11 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.523151896e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=229248 combine_payload_bytes=159244288 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159473536 ingress_bytes=209575936 link_bytes=209575936 token_bytes=318717824 weight_movement_bytes=50331648 total_nvlink_bytes=369049472 weight_to_token_ratio=0.157919 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594630721e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=38892 dup_rows=20988 dup_rows_remote=18240 dedup_ratio=0.320251 router_tokens_per_expert=[523, 483, 526, 485, 487, 538, 498, 514, 517, 487, 522, 541, 544, 514, 540, 493, 533, 514, 496, 540, 494, 550, 499, 533, 500, 487, 526, 527, 525, 521, 472, 513, 497, 494, 524, 540, 495, 506, 511, 504, 489, 497, 519, 521, 519, 504, 535, 562, 499, 499, 527, 466, 534, 527, 506, 496, 523, 529, 498, 508, 527, 522, 461, 487] payload_rows_per_dst=[5656, 5652, 5628, 5528, 5564, 5500, 5540, 5480] dup_rows_per_dst=[2748, 2672, 2698, 2368, 2528, 2776, 2568, 2630] combine_rows_per_src=[5656, 5652, 5628, 5528, 5564, 5500, 5540, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160350208 dispatch_meta_bytes=229720 combine_payload_bytes=160350208 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160579928 ingress_bytes=210681856 link_bytes=210681856 token_bytes=320930136 weight_movement_bytes=50331648 total_nvlink_bytes=371261784 weight_to_token_ratio=0.156831 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.603045495e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160604160 dispatch_meta_bytes=229080 combine_payload_bytes=160604160 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160833240 ingress_bytes=210935808 link_bytes=210935808 token_bytes=321437400 weight_movement_bytes=50331648 total_nvlink_bytes=371769048 weight_to_token_ratio=0.156583 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.604977776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5103978600 prefetch_bytes=352321536 reduce_bytes=327155712 weight_movement_bytes=679477248 egress_bytes=2553825384 ingress_bytes=3229630464 link_bytes=3229630464 total_nvlink_bytes=5783455848 weight_to_token_ratio=0.133127 prefetch_share_of_ingress=0.109090 dedup_ratio=0.321400 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=13 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.535859762e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5115545416 prefetch_bytes=377487360 reduce_bytes=201326592 weight_movement_bytes=578813952 egress_bytes=2559608648 ingress_bytes=3134750720 link_bytes=3134750720 total_nvlink_bytes=5694359368 weight_to_token_ratio=0.113148 prefetch_share_of_ingress=0.120420 dedup_ratio=0.320175 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=8 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.490739436e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159301632 dispatch_meta_bytes=228528 combine_payload_bytes=159301632 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159530160 ingress_bytes=209633280 link_bytes=209633280 token_bytes=318831792 weight_movement_bytes=50331648 total_nvlink_bytes=369163440 weight_to_token_ratio=0.157863 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595067043e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5115135384 prefetch_bytes=226492416 reduce_bytes=528482304 weight_movement_bytes=754974720 egress_bytes=2559403416 ingress_bytes=3310706688 link_bytes=3310706688 total_nvlink_bytes=5870110104 weight_to_token_ratio=0.147596 prefetch_share_of_ingress=0.068412 dedup_ratio=0.320316 prefetch_slots=18 unused_prefetch_slots=238 slot_utilization=0.070312 reduce_slots=21 remote_demand=9 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=19431 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.574415787e-04 prefetch_bytes_per_gemm_flop=1.077090993e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5095444048 prefetch_bytes=402653184 reduce_bytes=301989888 weight_movement_bytes=704643072 egress_bytes=2549550672 ingress_bytes=3250536448 link_bytes=3250536448 total_nvlink_bytes=5800087120 weight_to_token_ratio=0.138289 prefetch_share_of_ingress=0.123873 dedup_ratio=0.321409 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=12 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=20320 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.545801661e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none + [2026-09-21 08:32:49] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 212.8 | throughput per GPU (TFLOP/s/GPU): 151.4 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.118810E+00 | loss scale: 1.0 | grad norm: 11.194 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38954 dup_rows=21058 dup_rows_remote=18470 dedup_ratio=0.321320 router_tokens_per_expert=[480, 518, 551, 500, 518, 520, 538, 496, 518, 499, 492, 506, 484, 505, 500, 521, 499, 507, 513, 544, 528, 469, 491, 453, 524, 505, 487, 565, 563, 529, 495, 535, 492, 487, 573, 497, 506, 480, 517, 504, 498, 502, 562, 538, 488, 515, 511, 488, 521, 532, 509, 516, 512, 505, 520, 536, 528, 502, 497, 531, 544, 523, 475, 506] payload_rows_per_dst=[5622, 5494, 5454, 5650, 5524, 5592, 5606, 5536] dup_rows_per_dst=[2620, 2556, 2554, 2756, 2588, 2612, 2696, 2676] combine_rows_per_src=[5622, 5494, 5454, 5650, 5524, 5592, 5606, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=39022 dup_rows=21088 dup_rows_remote=18442 dedup_ratio=0.321777 router_tokens_per_expert=[513, 493, 516, 523, 543, 501, 515, 536, 493, 520, 544, 527, 498, 489, 533, 523, 541, 532, 546, 498, 534, 508, 513, 475, 515, 482, 511, 500, 573, 510, 500, 512, 539, 501, 530, 519, 495, 554, 523, 500, 488, 545, 493, 532, 505, 480, 517, 473, 500, 518, 515, 515, 522, 502, 466, 498, 468, 533, 547, 486, 472, 467, 512, 536] payload_rows_per_dst=[5584, 5672, 5550, 5590, 5618, 5494, 5426, 5514] dup_rows_per_dst=[2696, 2582, 2744, 2616, 2704, 2572, 2646, 2528] combine_rows_per_src=[5584, 5672, 5550, 5590, 5618, 5494, 5426, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44396 payload_rows_remote=38906 dup_rows=21140 dup_rows_remote=18514 dedup_ratio=0.322571 router_tokens_per_expert=[506, 519, 493, 505, 504, 543, 513, 484, 516, 494, 504, 509, 499, 553, 527, 524, 510, 496, 510, 507, 511, 475, 520, 529, 500, 532, 508, 509, 537, 534, 469, 512, 537, 487, 507, 506, 539, 536, 535, 477, 538, 522, 504, 478, 498, 541, 481, 524, 523, 507, 462, 535, 483, 527, 503, 500, 518, 504, 521, 532, 520, 527, 519, 525] payload_rows_per_dst=[5554, 5524, 5490, 5614, 5520, 5596, 5508, 5590] dup_rows_per_dst=[2580, 2728, 2626, 2588, 2728, 2576, 2572, 2742] combine_rows_per_src=[5554, 5524, 5490, 5614, 5520, 5596, 5508, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44580 payload_rows_remote=38996 dup_rows=20956 dup_rows_remote=18218 dedup_ratio=0.319763 router_tokens_per_expert=[506, 503, 518, 545, 507, 504, 514, 495, 526, 514, 501, 492, 526, 504, 467, 495, 537, 510, 518, 491, 493, 510, 482, 509, 517, 509, 522, 513, 529, 521, 514, 536, 529, 488, 526, 517, 505, 542, 536, 516, 566, 514, 523, 457, 514, 495, 501, 524, 506, 515, 526, 520, 512, 510, 485, 514, 503, 519, 578, 477, 488, 524, 492, 518] payload_rows_per_dst=[5566, 5602, 5466, 5584, 5660, 5572, 5566, 5564] dup_rows_per_dst=[2618, 2448, 2634, 2738, 2658, 2616, 2610, 2634] combine_rows_per_src=[5566, 5602, 5466, 5584, 5660, 5572, 5566, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=38906 dup_rows=21086 dup_rows_remote=18542 dedup_ratio=0.321747 router_tokens_per_expert=[562, 478, 505, 512, 470, 497, 501, 479, 475, 546, 492, 533, 503, 509, 486, 483, 537, 517, 546, 541, 514, 521, 477, 510, 549, 531, 524, 519, 521, 514, 509, 537, 489, 554, 539, 534, 490, 508, 520, 523, 519, 530, 522, 493, 516, 553, 519, 532, 489, 504, 515, 459, 531, 470, 508, 513, 469, 505, 534, 489, 505, 502, 512, 524] payload_rows_per_dst=[5478, 5508, 5600, 5384, 5614, 5782, 5540, 5544] dup_rows_per_dst=[2472, 2604, 2726, 2452, 2848, 2838, 2602, 2544] combine_rows_per_src=[5478, 5508, 5600, 5384, 5614, 5782, 5540, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=39012 dup_rows=21036 dup_rows_remote=18380 dedup_ratio=0.320984 router_tokens_per_expert=[523, 535, 511, 527, 535, 521, 540, 513, 526, 465, 512, 515, 532, 478, 515, 535, 507, 486, 517, 510, 497, 519, 573, 519, 509, 477, 555, 486, 535, 552, 521, 504, 496, 480, 485, 481, 493, 488, 497, 482, 490, 510, 497, 502, 515, 571, 482, 505, 523, 479, 558, 536, 537, 536, 491, 506, 509, 512, 502, 532, 495, 503, 516, 509] payload_rows_per_dst=[5610, 5538, 5578, 5646, 5416, 5488, 5658, 5566] dup_rows_per_dst=[2800, 2618, 2678, 2632, 2388, 2656, 2674, 2590] combine_rows_per_src=[5610, 5538, 5578, 5646, 5416, 5488, 5658, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44622 payload_rows_remote=38948 dup_rows=20914 dup_rows_remote=18190 dedup_ratio=0.319122 router_tokens_per_expert=[493, 468, 513, 540, 527, 523, 478, 537, 503, 551, 556, 509, 490, 580, 472, 538, 490, 530, 527, 518, 505, 523, 503, 526, 492, 499, 529, 533, 500, 537, 483, 508, 494, 457, 526, 461, 499, 535, 504, 506, 524, 489, 502, 530, 499, 494, 507, 474, 520, 525, 503, 526, 491, 519, 511, 515, 505, 535, 531, 534, 526, 505, 525, 515] payload_rows_per_dst=[5546, 5674, 5570, 5580, 5552, 5528, 5590, 5582] dup_rows_per_dst=[2612, 2724, 2674, 2582, 2412, 2510, 2630, 2770] combine_rows_per_src=[5546, 5674, 5570, 5580, 5552, 5528, 5590, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=229696 combine_payload_bytes=159555584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159785280 ingress_bytes=184721408 link_bytes=184721408 token_bytes=319340864 weight_movement_bytes=25165824 total_nvlink_bytes=344506688 weight_to_token_ratio=0.078806 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405516481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159834112 dispatch_meta_bytes=229856 combine_payload_bytes=159834112 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160063968 ingress_bytes=184999936 link_bytes=184999936 token_bytes=319898080 weight_movement_bytes=25165824 total_nvlink_bytes=345063904 weight_to_token_ratio=0.078668 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407635757e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229680 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159588656 ingress_bytes=184524800 link_bytes=184524800 token_bytes=318947632 weight_movement_bytes=25165824 total_nvlink_bytes=344113456 weight_to_token_ratio=0.078903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404020521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44286 payload_rows_remote=38762 dup_rows=21250 dup_rows_remote=18596 dedup_ratio=0.324249 router_tokens_per_expert=[495, 486, 517, 484, 490, 523, 570, 522, 545, 544, 497, 519, 533, 487, 514, 493, 491, 542, 505, 508, 504, 486, 526, 507, 514, 501, 563, 488, 506, 516, 502, 474, 522, 542, 541, 544, 516, 505, 488, 495, 521, 520, 494, 499, 501, 487, 504, 526, 508, 537, 489, 497, 500, 570, 500, 505, 502, 485, 530, 492, 530, 541, 520, 505] payload_rows_per_dst=[5524, 5584, 5510, 5494, 5538, 5456, 5584, 5596] dup_rows_per_dst=[2654, 2680, 2682, 2630, 2768, 2648, 2628, 2560] combine_rows_per_src=[5524, 5584, 5510, 5494, 5538, 5456, 5584, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159727616 dispatch_meta_bytes=228856 combine_payload_bytes=159727616 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=159956472 ingress_bytes=285556736 link_bytes=285556736 token_bytes=319684088 weight_movement_bytes=125829120 total_nvlink_bytes=445513208 weight_to_token_ratio=0.393605 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.172756818e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229792 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159588768 ingress_bytes=209690624 link_bytes=209690624 token_bytes=318947744 weight_movement_bytes=50331648 total_nvlink_bytes=369279392 weight_to_token_ratio=0.157805 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595503365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159793152 dispatch_meta_bytes=229568 combine_payload_bytes=159793152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160022720 ingress_bytes=184958976 link_bytes=184958976 token_bytes=319815872 weight_movement_bytes=25165824 total_nvlink_bytes=344981696 weight_to_token_ratio=0.078688 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407324099e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159531008 dispatch_meta_bytes=228552 combine_payload_bytes=159531008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159759560 ingress_bytes=184696832 link_bytes=184696832 token_bytes=319290568 weight_movement_bytes=25165824 total_nvlink_bytes=344456392 weight_to_token_ratio=0.078818 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405329486e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158769152 dispatch_meta_bytes=229432 combine_payload_bytes=158769152 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158998584 ingress_bytes=209100800 link_bytes=209100800 token_bytes=317767736 weight_movement_bytes=50331648 total_nvlink_bytes=368099384 weight_to_token_ratio=0.158391 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591015485e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44442 payload_rows_remote=38922 dup_rows=21094 dup_rows_remote=18570 dedup_ratio=0.321869 router_tokens_per_expert=[528, 518, 511, 513, 527, 519, 494, 504, 494, 485, 514, 500, 515, 521, 525, 506, 502, 506, 460, 538, 530, 532, 478, 528, 530, 517, 486, 525, 499, 503, 503, 506, 500, 517, 485, 525, 529, 493, 497, 476, 523, 503, 516, 498, 509, 510, 512, 544, 522, 514, 531, 528, 521, 528, 542, 532, 498, 523, 505, 518, 468, 528, 509, 547] payload_rows_per_dst=[5530, 5528, 5478, 5498, 5520, 5626, 5682, 5580] dup_rows_per_dst=[2698, 2592, 2670, 2640, 2524, 2604, 2754, 2612] combine_rows_per_src=[5530, 5528, 5478, 5498, 5520, 5626, 5682, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44470 payload_rows_remote=38948 dup_rows=21066 dup_rows_remote=18382 dedup_ratio=0.321442 router_tokens_per_expert=[518, 530, 459, 503, 514, 523, 506, 522, 523, 516, 519, 552, 506, 503, 503, 517, 513, 508, 486, 507, 513, 538, 499, 509, 483, 508, 510, 508, 513, 499, 520, 483, 518, 514, 512, 539, 520, 516, 495, 544, 489, 459, 520, 535, 510, 462, 525, 547, 504, 483, 539, 516, 522, 529, 512, 498, 541, 492, 536, 535, 520, 459, 503, 563] payload_rows_per_dst=[5554, 5642, 5586, 5508, 5540, 5546, 5522, 5572] dup_rows_per_dst=[2596, 2636, 2560, 2540, 2776, 2548, 2684, 2726] combine_rows_per_src=[5554, 5642, 5586, 5508, 5540, 5546, 5522, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44368 payload_rows_remote=38860 dup_rows=21168 dup_rows_remote=18532 dedup_ratio=0.322998 router_tokens_per_expert=[535, 528, 525, 483, 529, 521, 509, 517, 519, 484, 490, 492, 511, 475, 516, 489, 517, 448, 515, 523, 503, 531, 538, 497, 499, 499, 487, 530, 536, 507, 525, 506, 546, 508, 527, 509, 508, 512, 507, 513, 484, 513, 518, 508, 488, 499, 510, 515, 522, 471, 507, 522, 506, 544, 511, 510, 512, 532, 543, 542, 517, 527, 521, 532] payload_rows_per_dst=[5650, 5514, 5508, 5484, 5582, 5480, 5506, 5644] dup_rows_per_dst=[2644, 2438, 2636, 2694, 2678, 2590, 2680, 2808] combine_rows_per_src=[5650, 5514, 5508, 5484, 5582, 5480, 5506, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44528 payload_rows_remote=39014 dup_rows=21008 dup_rows_remote=18500 dedup_ratio=0.320557 router_tokens_per_expert=[541, 535, 517, 519, 472, 503, 506, 486, 534, 498, 496, 528, 520, 482, 474, 527, 493, 493, 498, 509, 535, 531, 514, 535, 488, 546, 504, 496, 539, 499, 513, 524, 520, 534, 533, 523, 536, 553, 496, 506, 517, 503, 521, 524, 479, 501, 511, 532, 556, 525, 504, 503, 482, 527, 508, 508, 479, 487, 513, 541, 513, 515, 488, 475] payload_rows_per_dst=[5384, 5476, 5568, 5662, 5710, 5612, 5602, 5514] dup_rows_per_dst=[2368, 2642, 2648, 2962, 2662, 2564, 2654, 2508] combine_rows_per_src=[5384, 5476, 5568, 5662, 5710, 5612, 5602, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38954 dup_rows=21034 dup_rows_remote=18388 dedup_ratio=0.320953 router_tokens_per_expert=[495, 502, 484, 518, 508, 521, 546, 528, 524, 499, 498, 545, 478, 542, 515, 495, 508, 528, 492, 512, 502, 486, 519, 516, 479, 513, 524, 534, 502, 540, 482, 523, 504, 523, 529, 517, 517, 536, 505, 522, 497, 485, 515, 519, 509, 506, 505, 520, 513, 510, 506, 520, 523, 535, 527, 507, 520, 530, 512, 484, 481, 530, 515, 488] payload_rows_per_dst=[5656, 5608, 5512, 5548, 5584, 5502, 5514, 5578] dup_rows_per_dst=[2548, 2584, 2614, 2646, 2722, 2610, 2768, 2542] combine_rows_per_src=[5656, 5608, 5512, 5548, 5584, 5502, 5514, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44310 payload_rows_remote=38788 dup_rows=21226 dup_rows_remote=18734 dedup_ratio=0.323883 router_tokens_per_expert=[520, 500, 518, 488, 508, 537, 527, 483, 508, 486, 522, 502, 508, 522, 494, 524, 512, 502, 499, 494, 477, 481, 546, 513, 546, 509, 493, 490, 498, 536, 520, 499, 502, 517, 496, 506, 512, 514, 511, 517, 477, 544, 486, 495, 466, 512, 519, 508, 535, 536, 500, 533, 510, 518, 539, 520, 506, 551, 517, 535, 529, 543, 514, 538] payload_rows_per_dst=[5546, 5474, 5440, 5560, 5566, 5522, 5552, 5650] dup_rows_per_dst=[2616, 2658, 2608, 2622, 2584, 2492, 2830, 2816] combine_rows_per_src=[5546, 5474, 5440, 5560, 5566, 5522, 5552, 5650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44492 payload_rows_remote=39014 dup_rows=21044 dup_rows_remote=18370 dedup_ratio=0.321106 router_tokens_per_expert=[546, 510, 510, 493, 514, 519, 593, 510, 480, 519, 487, 531, 514, 496, 497, 552, 523, 529, 525, 507, 506, 490, 537, 496, 543, 465, 539, 522, 477, 541, 477, 507, 484, 550, 501, 564, 506, 516, 479, 476, 515, 497, 494, 509, 504, 493, 515, 511, 535, 528, 489, 512, 490, 518, 502, 510, 498, 534, 524, 523, 507, 503, 527, 499] payload_rows_per_dst=[5708, 5478, 5596, 5544, 5574, 5504, 5500, 5588] dup_rows_per_dst=[2682, 2674, 2630, 2598, 2578, 2572, 2668, 2642] combine_rows_per_src=[5708, 5478, 5596, 5544, 5574, 5504, 5500, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=229968 combine_payload_bytes=159424512 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159654480 ingress_bytes=234921984 link_bytes=234921984 token_bytes=319078992 weight_movement_bytes=75497472 total_nvlink_bytes=394576464 weight_to_token_ratio=0.236611 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787484861e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159531008 dispatch_meta_bytes=229320 combine_payload_bytes=159531008 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159760328 ingress_bytes=209862656 link_bytes=209862656 token_bytes=319291336 weight_movement_bytes=50331648 total_nvlink_bytes=369622984 weight_to_token_ratio=0.157635 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596812329e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159170560 dispatch_meta_bytes=229568 combine_payload_bytes=159170560 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159400128 ingress_bytes=184336384 link_bytes=184336384 token_bytes=318570688 weight_movement_bytes=25165824 total_nvlink_bytes=343736512 weight_to_token_ratio=0.078996 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402586893e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159801344 dispatch_meta_bytes=230056 combine_payload_bytes=159801344 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160031400 ingress_bytes=235298816 link_bytes=235298816 token_bytes=319832744 weight_movement_bytes=75497472 total_nvlink_bytes=395330216 weight_to_token_ratio=0.236053 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790352117e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=229368 combine_payload_bytes=159555584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159784952 ingress_bytes=184721408 link_bytes=184721408 token_bytes=319340536 weight_movement_bytes=25165824 total_nvlink_bytes=344506360 weight_to_token_ratio=0.078806 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405516481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158875648 dispatch_meta_bytes=230088 combine_payload_bytes=158875648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159105736 ingress_bytes=184041472 link_bytes=184041472 token_bytes=317981384 weight_movement_bytes=25165824 total_nvlink_bytes=343147208 weight_to_token_ratio=0.079142 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400342954e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44408 payload_rows_remote=38762 dup_rows=21128 dup_rows_remote=18418 dedup_ratio=0.322388 router_tokens_per_expert=[502, 515, 509, 527, 486, 557, 538, 507, 515, 486, 520, 494, 562, 534, 481, 496, 498, 519, 500, 514, 527, 545, 484, 531, 495, 477, 496, 525, 479, 520, 535, 488, 483, 524, 477, 512, 513, 514, 522, 544, 514, 499, 521, 491, 519, 496, 510, 527, 514, 496, 513, 578, 499, 493, 518, 504, 536, 514, 536, 488, 542, 501, 493, 515] payload_rows_per_dst=[5646, 5746, 5610, 5474, 5494, 5782, 5216, 5440] dup_rows_per_dst=[2710, 2846, 2872, 2556, 2442, 2982, 2404, 2316] combine_rows_per_src=[5646, 5746, 5610, 5474, 5494, 5782, 5216, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159801344 dispatch_meta_bytes=229536 combine_payload_bytes=159801344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160030880 ingress_bytes=184967168 link_bytes=184967168 token_bytes=319832224 weight_movement_bytes=25165824 total_nvlink_bytes=344998048 weight_to_token_ratio=0.078684 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407386431e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44334 payload_rows_remote=38686 dup_rows=21202 dup_rows_remote=18408 dedup_ratio=0.323517 router_tokens_per_expert=[499, 493, 539, 523, 463, 506, 537, 514, 507, 499, 495, 498, 509, 506, 531, 514, 541, 529, 498, 518, 478, 470, 523, 490, 518, 524, 501, 502, 478, 535, 499, 492, 514, 568, 504, 528, 537, 554, 475, 541, 499, 525, 476, 551, 535, 542, 568, 527, 518, 509, 535, 450, 475, 530, 507, 534, 461, 520, 506, 505, 469, 538, 532, 506] payload_rows_per_dst=[5558, 5550, 5490, 5532, 5648, 5602, 5482, 5472] dup_rows_per_dst=[2590, 2568, 2604, 2566, 2794, 2844, 2634, 2602] combine_rows_per_src=[5558, 5550, 5490, 5532, 5648, 5602, 5482, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38834 dup_rows=21088 dup_rows_remote=18518 dedup_ratio=0.321777 router_tokens_per_expert=[544, 533, 512, 490, 545, 486, 523, 549, 516, 497, 532, 489, 545, 507, 504, 499, 476, 497, 487, 499, 502, 509, 465, 502, 497, 501, 530, 559, 498, 496, 523, 511, 516, 538, 480, 522, 538, 545, 505, 495, 522, 504, 485, 522, 495, 484, 548, 517, 506, 537, 483, 507, 495, 505, 558, 501, 542, 497, 517, 493, 522, 501, 549, 516] payload_rows_per_dst=[5596, 5550, 5406, 5572, 5600, 5508, 5614, 5602] dup_rows_per_dst=[2768, 2628, 2468, 2658, 2678, 2646, 2570, 2672] combine_rows_per_src=[5596, 5550, 5406, 5572, 5600, 5508, 5614, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158769152 dispatch_meta_bytes=228720 combine_payload_bytes=158769152 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158997872 ingress_bytes=209100800 link_bytes=209100800 token_bytes=317767024 weight_movement_bytes=50331648 total_nvlink_bytes=368098672 weight_to_token_ratio=0.158392 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591015485e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44496 payload_rows_remote=38910 dup_rows=21040 dup_rows_remote=18378 dedup_ratio=0.321045 router_tokens_per_expert=[494, 486, 502, 526, 526, 516, 514, 505, 522, 506, 503, 525, 464, 526, 529, 489, 498, 517, 520, 514, 531, 544, 489, 511, 514, 491, 515, 501, 536, 535, 542, 487, 499, 482, 519, 512, 495, 514, 522, 522, 495, 507, 514, 512, 531, 464, 577, 503, 518, 470, 476, 538, 504, 515, 526, 522, 535, 499, 531, 570, 483, 502, 511, 522] payload_rows_per_dst=[5488, 5540, 5586, 5590, 5570, 5566, 5552, 5604] dup_rows_per_dst=[2650, 2588, 2662, 2652, 2560, 2640, 2586, 2702] combine_rows_per_src=[5488, 5540, 5586, 5590, 5570, 5566, 5552, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44308 payload_rows_remote=38726 dup_rows=21228 dup_rows_remote=18614 dedup_ratio=0.323914 router_tokens_per_expert=[503, 481, 503, 495, 522, 493, 509, 513, 527, 496, 551, 547, 503, 483, 547, 508, 533, 507, 508, 517, 479, 498, 546, 542, 560, 494, 481, 528, 542, 499, 491, 506, 561, 507, 508, 495, 511, 483, 507, 512, 517, 522, 468, 529, 499, 468, 520, 504, 518, 535, 556, 536, 518, 501, 502, 491, 505, 501, 533, 496, 515, 513, 534, 491] payload_rows_per_dst=[5418, 5558, 5596, 5542, 5356, 5520, 5736, 5582] dup_rows_per_dst=[2440, 2766, 2782, 2640, 2420, 2596, 2970, 2614] combine_rows_per_src=[5418, 5558, 5596, 5542, 5356, 5520, 5736, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44570 payload_rows_remote=38952 dup_rows=20966 dup_rows_remote=18368 dedup_ratio=0.319916 router_tokens_per_expert=[555, 545, 472, 510, 565, 521, 495, 502, 529, 507, 509, 479, 508, 545, 494, 519, 484, 531, 527, 479, 517, 537, 501, 487, 553, 503, 526, 514, 524, 497, 507, 484, 510, 549, 532, 539, 514, 484, 524, 511, 518, 512, 570, 514, 494, 472, 487, 522, 495, 554, 484, 499, 519, 459, 483, 525, 503, 528, 469, 530, 491, 500, 550, 501] payload_rows_per_dst=[5570, 5560, 5556, 5618, 5678, 5548, 5476, 5564] dup_rows_per_dst=[2760, 2620, 2570, 2598, 2648, 2630, 2560, 2580] combine_rows_per_src=[5570, 5560, 5556, 5618, 5678, 5548, 5476, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38864 dup_rows=21092 dup_rows_remote=18466 dedup_ratio=0.321838 router_tokens_per_expert=[469, 504, 518, 537, 495, 517, 542, 545, 519, 487, 518, 504, 553, 517, 519, 522, 527, 512, 488, 501, 528, 522, 526, 546, 500, 534, 476, 512, 523, 512, 509, 508, 496, 528, 488, 489, 531, 506, 538, 497, 491, 495, 538, 494, 533, 489, 543, 520, 476, 524, 486, 487, 463, 530, 533, 499, 469, 523, 522, 512, 521, 502, 556, 499] payload_rows_per_dst=[5596, 5612, 5502, 5520, 5520, 5580, 5516, 5598] dup_rows_per_dst=[2658, 2666, 2798, 2628, 2626, 2626, 2480, 2610] combine_rows_per_src=[5596, 5612, 5502, 5520, 5520, 5580, 5516, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158457856 dispatch_meta_bytes=228376 combine_payload_bytes=158457856 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158686232 ingress_bytes=208789504 link_bytes=208789504 token_bytes=317144088 weight_movement_bytes=50331648 total_nvlink_bytes=367475736 weight_to_token_ratio=0.158703 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588646882e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159064064 dispatch_meta_bytes=229408 combine_payload_bytes=159064064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159293472 ingress_bytes=184229888 link_bytes=184229888 token_bytes=318357536 weight_movement_bytes=25165824 total_nvlink_bytes=343523360 weight_to_token_ratio=0.079049 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401776582e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44398 payload_rows_remote=38842 dup_rows=21138 dup_rows_remote=18436 dedup_ratio=0.322540 router_tokens_per_expert=[481, 517, 504, 488, 534, 543, 490, 523, 531, 505, 511, 533, 503, 496, 498, 552, 529, 519, 498, 534, 521, 514, 512, 494, 486, 568, 525, 520, 501, 537, 522, 482, 535, 516, 520, 502, 474, 495, 489, 515, 487, 543, 507, 486, 515, 527, 501, 488, 511, 517, 496, 508, 523, 479, 491, 546, 498, 514, 508, 514, 534, 517, 527, 514] payload_rows_per_dst=[5576, 5556, 5562, 5616, 5496, 5506, 5530, 5556] dup_rows_per_dst=[2584, 2702, 2680, 2666, 2596, 2602, 2612, 2696] combine_rows_per_src=[5576, 5556, 5562, 5616, 5496, 5506, 5530, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158621696 dispatch_meta_bytes=229360 combine_payload_bytes=158621696 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158851056 ingress_bytes=208953344 link_bytes=208953344 token_bytes=317472752 weight_movement_bytes=50331648 total_nvlink_bytes=367804400 weight_to_token_ratio=0.158538 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589893516e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159186944 dispatch_meta_bytes=229320 combine_payload_bytes=159186944 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159416264 ingress_bytes=184352768 link_bytes=184352768 token_bytes=318603208 weight_movement_bytes=25165824 total_nvlink_bytes=343769032 weight_to_token_ratio=0.078988 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402711557e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159375360 dispatch_meta_bytes=229152 combine_payload_bytes=159375360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159604512 ingress_bytes=184541184 link_bytes=184541184 token_bytes=318979872 weight_movement_bytes=25165824 total_nvlink_bytes=344145696 weight_to_token_ratio=0.078895 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404145185e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159547392 dispatch_meta_bytes=229280 combine_payload_bytes=159547392 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159776672 ingress_bytes=209879040 link_bytes=209879040 token_bytes=319324064 weight_movement_bytes=50331648 total_nvlink_bytes=369655712 weight_to_token_ratio=0.157619 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596936993e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38896 dup_rows=21050 dup_rows_remote=18356 dedup_ratio=0.321198 router_tokens_per_expert=[495, 565, 476, 534, 524, 478, 549, 507, 467, 497, 521, 511, 536, 515, 478, 523, 508, 521, 509, 529, 513, 506, 536, 515, 513, 507, 515, 473, 498, 499, 545, 521, 523, 491, 531, 490, 481, 516, 500, 542, 503, 532, 493, 510, 496, 516, 516, 495, 538, 533, 496, 518, 507, 504, 510, 503, 516, 485, 547, 531, 524, 549, 486, 502] payload_rows_per_dst=[5590, 5460, 5638, 5536, 5628, 5530, 5568, 5536] dup_rows_per_dst=[2694, 2612, 2636, 2602, 2718, 2592, 2650, 2546] combine_rows_per_src=[5590, 5460, 5638, 5536, 5628, 5530, 5568, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159096832 dispatch_meta_bytes=229112 combine_payload_bytes=159096832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159325944 ingress_bytes=209428480 link_bytes=209428480 token_bytes=318422776 weight_movement_bytes=50331648 total_nvlink_bytes=368754424 weight_to_token_ratio=0.158065 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593508752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=38990 dup_rows=20946 dup_rows_remote=18222 dedup_ratio=0.319611 router_tokens_per_expert=[523, 483, 482, 523, 552, 451, 527, 505, 531, 525, 509, 503, 497, 490, 512, 494, 523, 497, 501, 541, 536, 568, 508, 504, 521, 534, 501, 530, 468, 500, 535, 516, 498, 511, 504, 490, 519, 530, 533, 536, 520, 520, 514, 497, 489, 501, 525, 513, 475, 518, 540, 541, 539, 508, 555, 486, 503, 512, 499, 538, 470, 502, 490, 502] payload_rows_per_dst=[5604, 5494, 5666, 5528, 5622, 5550, 5600, 5526] dup_rows_per_dst=[2488, 2628, 2690, 2682, 2620, 2608, 2724, 2506] combine_rows_per_src=[5604, 5494, 5666, 5528, 5622, 5550, 5600, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44386 payload_rows_remote=38868 dup_rows=21150 dup_rows_remote=18546 dedup_ratio=0.322723 router_tokens_per_expert=[507, 523, 489, 487, 552, 515, 526, 483, 496, 525, 506, 507, 513, 503, 526, 529, 510, 550, 499, 592, 525, 525, 496, 529, 531, 480, 496, 492, 533, 461, 507, 526, 555, 487, 525, 474, 510, 518, 487, 505, 493, 496, 494, 477, 525, 542, 494, 535, 518, 538, 507, 479, 546, 513, 487, 508, 515, 510, 538, 513, 513, 521, 514, 492] payload_rows_per_dst=[5562, 5556, 5642, 5450, 5518, 5492, 5586, 5580] dup_rows_per_dst=[2602, 2654, 2810, 2602, 2604, 2620, 2606, 2652] combine_rows_per_src=[5562, 5556, 5642, 5450, 5518, 5492, 5586, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159318016 dispatch_meta_bytes=229008 combine_payload_bytes=159318016 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159547024 ingress_bytes=234815488 link_bytes=234815488 token_bytes=318865040 weight_movement_bytes=75497472 total_nvlink_bytes=394362512 weight_to_token_ratio=0.236769 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786674549e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44458 payload_rows_remote=38692 dup_rows=21078 dup_rows_remote=18108 dedup_ratio=0.321625 router_tokens_per_expert=[497, 488, 532, 478, 488, 500, 545, 492, 506, 477, 493, 524, 551, 514, 541, 510, 480, 514, 501, 547, 505, 485, 496, 487, 539, 528, 517, 525, 498, 501, 515, 489, 519, 488, 526, 505, 524, 515, 517, 499, 528, 490, 553, 514, 483, 530, 515, 489, 502, 527, 513, 509, 509, 533, 531, 522, 543, 520, 535, 541, 499, 529, 521, 476] payload_rows_per_dst=[5514, 5430, 5194, 5816, 5542, 5696, 5500, 5766] dup_rows_per_dst=[2526, 2366, 2158, 3044, 2644, 2944, 2426, 2970] combine_rows_per_src=[5514, 5430, 5194, 5816, 5542, 5696, 5500, 5766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44406 payload_rows_remote=38984 dup_rows=21130 dup_rows_remote=18644 dedup_ratio=0.322418 router_tokens_per_expert=[524, 482, 527, 520, 489, 493, 508, 500, 500, 497, 524, 561, 526, 508, 546, 549, 489, 523, 528, 539, 505, 477, 481, 493, 483, 465, 542, 553, 522, 474, 517, 508, 531, 550, 513, 517, 548, 534, 463, 544, 505, 508, 524, 496, 490, 478, 472, 481, 534, 535, 518, 517, 542, 523, 516, 527, 516, 514, 513, 479, 509, 534, 465, 519] payload_rows_per_dst=[5446, 5654, 5540, 5458, 5664, 5422, 5716, 5506] dup_rows_per_dst=[2640, 2768, 2530, 2670, 2736, 2486, 2708, 2592] combine_rows_per_src=[5446, 5654, 5540, 5458, 5664, 5422, 5716, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159703040 dispatch_meta_bytes=228848 combine_payload_bytes=159703040 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159931888 ingress_bytes=235200512 link_bytes=235200512 token_bytes=319634928 weight_movement_bytes=75497472 total_nvlink_bytes=395132400 weight_to_token_ratio=0.236199 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789604137e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38924 dup_rows=21000 dup_rows_remote=18220 dedup_ratio=0.320435 router_tokens_per_expert=[515, 549, 478, 527, 485, 500, 519, 513, 522, 503, 540, 506, 513, 509, 525, 490, 554, 535, 512, 504, 503, 545, 506, 537, 518, 485, 516, 515, 510, 497, 488, 504, 508, 500, 492, 523, 499, 503, 526, 506, 510, 520, 497, 472, 523, 514, 522, 506, 506, 502, 507, 564, 539, 535, 490, 520, 546, 498, 492, 538, 479, 498, 505, 505] payload_rows_per_dst=[5606, 5468, 5612, 5552, 5528, 5568, 5666, 5536] dup_rows_per_dst=[2566, 2748, 2780, 2514, 2586, 2560, 2660, 2586] combine_rows_per_src=[5606, 5468, 5612, 5552, 5528, 5568, 5666, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38924 dup_rows=21072 dup_rows_remote=18466 dedup_ratio=0.321533 router_tokens_per_expert=[551, 477, 551, 523, 490, 503, 503, 475, 523, 489, 514, 478, 522, 503, 506, 525, 512, 500, 506, 527, 523, 511, 514, 545, 498, 519, 508, 503, 517, 520, 527, 481, 517, 517, 498, 483, 539, 548, 517, 478, 528, 512, 488, 489, 508, 543, 518, 515, 563, 555, 551, 533, 525, 497, 510, 491, 506, 486, 509, 511, 460, 526, 507, 496] payload_rows_per_dst=[5498, 5502, 5646, 5540, 5550, 5510, 5656, 5562] dup_rows_per_dst=[2648, 2618, 2630, 2606, 2644, 2692, 2794, 2440] combine_rows_per_src=[5498, 5502, 5646, 5540, 5550, 5510, 5656, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159203328 dispatch_meta_bytes=229656 combine_payload_bytes=159203328 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159432984 ingress_bytes=184369152 link_bytes=184369152 token_bytes=318636312 weight_movement_bytes=25165824 total_nvlink_bytes=343802136 weight_to_token_ratio=0.078980 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402836220e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44616 payload_rows_remote=39056 dup_rows=20920 dup_rows_remote=18292 dedup_ratio=0.319214 router_tokens_per_expert=[512, 548, 496, 491, 521, 547, 475, 531, 531, 521, 527, 479, 497, 535, 501, 503, 542, 469, 511, 519, 514, 564, 514, 532, 478, 493, 501, 498, 504, 497, 521, 524, 509, 532, 509, 507, 511, 485, 522, 532, 456, 532, 513, 529, 521, 523, 495, 481, 516, 507, 498, 509, 515, 576, 505, 526, 519, 515, 507, 482, 487, 506, 509, 538] payload_rows_per_dst=[5578, 5560, 5650, 5504, 5552, 5562, 5646, 5564] dup_rows_per_dst=[2664, 2628, 2680, 2528, 2662, 2538, 2658, 2562] combine_rows_per_src=[5578, 5560, 5650, 5504, 5552, 5562, 5646, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158482432 dispatch_meta_bytes=227200 combine_payload_bytes=158482432 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158709632 ingress_bytes=183648256 link_bytes=183648256 token_bytes=317192064 weight_movement_bytes=25165824 total_nvlink_bytes=342357888 weight_to_token_ratio=0.079339 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397351034e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159678464 dispatch_meta_bytes=230512 combine_payload_bytes=159678464 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159908976 ingress_bytes=184844288 link_bytes=184844288 token_bytes=319587440 weight_movement_bytes=25165824 total_nvlink_bytes=344753264 weight_to_token_ratio=0.078745 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406451456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38720 dup_rows=21052 dup_rows_remote=18222 dedup_ratio=0.321228 router_tokens_per_expert=[515, 510, 472, 491, 492, 498, 492, 491, 519, 558, 487, 511, 521, 532, 557, 528, 574, 526, 528, 511, 492, 476, 506, 552, 525, 541, 466, 520, 516, 475, 469, 509, 488, 522, 524, 524, 481, 520, 464, 516, 525, 543, 531, 501, 522, 506, 502, 514, 483, 535, 535, 525, 522, 517, 525, 489, 514, 503, 509, 543, 500, 494, 518, 513] payload_rows_per_dst=[5764, 5608, 5558, 5502, 5466, 5644, 5356, 5586] dup_rows_per_dst=[2830, 2854, 2750, 2540, 2598, 2644, 2234, 2602] combine_rows_per_src=[5764, 5608, 5558, 5502, 5466, 5644, 5356, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=228576 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159661280 ingress_bytes=260096000 link_bytes=260096000 token_bytes=319093984 weight_movement_bytes=100663296 total_nvlink_bytes=419757280 weight_to_token_ratio=0.315466 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.979030036e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=229560 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159662264 ingress_bytes=184598528 link_bytes=184598528 token_bytes=319094968 weight_movement_bytes=25165824 total_nvlink_bytes=344260792 weight_to_token_ratio=0.078866 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404581506e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159973376 dispatch_meta_bytes=229392 combine_payload_bytes=159973376 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160202768 ingress_bytes=210305024 link_bytes=210305024 token_bytes=320176144 weight_movement_bytes=50331648 total_nvlink_bytes=370507792 weight_to_token_ratio=0.157200 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600178239e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44582 payload_rows_remote=39060 dup_rows=20954 dup_rows_remote=18284 dedup_ratio=0.319733 router_tokens_per_expert=[497, 532, 511, 526, 557, 525, 510, 513, 569, 492, 513, 478, 515, 488, 495, 502, 520, 523, 539, 504, 519, 509, 526, 479, 538, 487, 499, 481, 521, 485, 521, 502, 513, 494, 506, 542, 511, 484, 514, 552, 519, 532, 465, 515, 484, 525, 510, 484, 499, 531, 508, 498, 518, 505, 500, 537, 526, 514, 501, 562, 511, 492, 514, 526] payload_rows_per_dst=[5672, 5482, 5552, 5520, 5638, 5504, 5522, 5692] dup_rows_per_dst=[2670, 2622, 2686, 2548, 2594, 2564, 2670, 2600] combine_rows_per_src=[5672, 5482, 5552, 5520, 5638, 5504, 5522, 5692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158597120 dispatch_meta_bytes=227768 combine_payload_bytes=158597120 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158824888 ingress_bytes=183762944 link_bytes=183762944 token_bytes=317422008 weight_movement_bytes=25165824 total_nvlink_bytes=342587832 weight_to_token_ratio=0.079282 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398223677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44304 payload_rows_remote=38708 dup_rows=21232 dup_rows_remote=18530 dedup_ratio=0.323975 router_tokens_per_expert=[485, 482, 533, 485, 509, 471, 433, 507, 496, 539, 526, 526, 511, 477, 563, 486, 528, 499, 490, 520, 510, 546, 538, 476, 503, 524, 521, 552, 536, 520, 474, 513, 539, 522, 490, 511, 492, 559, 523, 513, 526, 494, 488, 489, 495, 485, 516, 508, 516, 518, 537, 539, 491, 540, 519, 534, 565, 514, 495, 500, 501, 523, 539, 508] payload_rows_per_dst=[5450, 5520, 5574, 5612, 5596, 5440, 5568, 5544] dup_rows_per_dst=[2360, 2728, 2640, 2674, 2702, 2562, 2820, 2746] combine_rows_per_src=[5450, 5520, 5574, 5612, 5596, 5440, 5568, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44604 payload_rows_remote=39014 dup_rows=20932 dup_rows_remote=18182 dedup_ratio=0.319397 router_tokens_per_expert=[493, 511, 502, 487, 470, 518, 522, 517, 539, 513, 492, 511, 526, 534, 498, 498, 524, 482, 510, 514, 517, 516, 499, 470, 522, 519, 463, 488, 518, 506, 532, 539, 489, 532, 512, 523, 509, 519, 508, 521, 494, 541, 502, 530, 527, 511, 491, 503, 513, 563, 488, 553, 503, 530, 481, 542, 505, 519, 544, 517, 505, 514, 523, 506] payload_rows_per_dst=[5566, 5572, 5566, 5562, 5588, 5562, 5598, 5590] dup_rows_per_dst=[2474, 2650, 2498, 2536, 2714, 2636, 2674, 2750] combine_rows_per_src=[5566, 5572, 5566, 5562, 5588, 5562, 5598, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38916 dup_rows=21088 dup_rows_remote=18582 dedup_ratio=0.321777 router_tokens_per_expert=[508, 485, 523, 519, 536, 516, 505, 492, 475, 516, 505, 512, 517, 463, 504, 518, 493, 534, 533, 508, 489, 504, 517, 540, 488, 512, 511, 532, 558, 537, 543, 513, 476, 557, 525, 489, 525, 489, 506, 521, 493, 486, 502, 513, 521, 484, 505, 515, 516, 532, 516, 492, 501, 508, 527, 531, 522, 511, 491, 494, 540, 542, 557, 475] payload_rows_per_dst=[5532, 5498, 5598, 5620, 5616, 5532, 5518, 5534] dup_rows_per_dst=[2636, 2522, 2638, 2768, 2560, 2506, 2728, 2730] combine_rows_per_src=[5532, 5498, 5598, 5620, 5616, 5532, 5518, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159989760 dispatch_meta_bytes=229376 combine_payload_bytes=159989760 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=160219136 ingress_bytes=260653056 link_bytes=260653056 token_bytes=320208896 weight_movement_bytes=100663296 total_nvlink_bytes=420872192 weight_to_token_ratio=0.314368 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.983268588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38888 dup_rows=21146 dup_rows_remote=18478 dedup_ratio=0.322662 router_tokens_per_expert=[519, 490, 526, 504, 503, 519, 513, 490, 513, 500, 522, 539, 488, 518, 503, 502, 515, 536, 517, 527, 513, 527, 529, 522, 468, 491, 549, 495, 520, 497, 536, 494, 553, 503, 518, 518, 505, 533, 491, 515, 510, 497, 493, 535, 549, 538, 475, 507, 530, 512, 546, 524, 545, 504, 519, 487, 478, 523, 499, 498, 524, 474, 499, 481] payload_rows_per_dst=[5518, 5502, 5690, 5490, 5594, 5478, 5660, 5458] dup_rows_per_dst=[2610, 2668, 2682, 2610, 2678, 2730, 2674, 2494] combine_rows_per_src=[5518, 5502, 5690, 5490, 5594, 5478, 5660, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38842 dup_rows=21122 dup_rows_remote=18508 dedup_ratio=0.322296 router_tokens_per_expert=[510, 544, 534, 499, 534, 518, 521, 537, 538, 520, 525, 488, 464, 509, 519, 505, 501, 486, 524, 529, 492, 525, 516, 520, 523, 559, 478, 514, 536, 468, 486, 518, 517, 533, 500, 510, 478, 546, 484, 527, 516, 528, 469, 524, 540, 497, 534, 532, 481, 536, 453, 482, 486, 561, 530, 486, 499, 495, 536, 488, 537, 502, 502, 519] payload_rows_per_dst=[5582, 5478, 5572, 5546, 5500, 5630, 5560, 5546] dup_rows_per_dst=[2812, 2658, 2614, 2618, 2690, 2650, 2470, 2610] combine_rows_per_src=[5582, 5478, 5572, 5546, 5500, 5630, 5560, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44376 payload_rows_remote=38784 dup_rows=21160 dup_rows_remote=18556 dedup_ratio=0.322876 router_tokens_per_expert=[509, 487, 509, 544, 488, 527, 480, 505, 564, 496, 526, 513, 499, 534, 511, 539, 506, 479, 524, 513, 545, 534, 485, 533, 521, 519, 487, 487, 544, 506, 522, 512, 546, 484, 496, 495, 474, 523, 525, 508, 527, 496, 545, 510, 519, 462, 478, 556, 529, 517, 461, 517, 500, 516, 570, 546, 472, 459, 506, 545, 480, 545, 507, 506] payload_rows_per_dst=[5534, 5562, 5566, 5592, 5598, 5512, 5564, 5448] dup_rows_per_dst=[2564, 2802, 2672, 2604, 2504, 2674, 2748, 2592] combine_rows_per_src=[5534, 5562, 5566, 5592, 5598, 5512, 5564, 5448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158547968 dispatch_meta_bytes=228952 combine_payload_bytes=158547968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158776920 ingress_bytes=183713792 link_bytes=183713792 token_bytes=317324888 weight_movement_bytes=25165824 total_nvlink_bytes=342490712 weight_to_token_ratio=0.079306 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397849688e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159399936 dispatch_meta_bytes=229992 combine_payload_bytes=159399936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159629928 ingress_bytes=184565760 link_bytes=184565760 token_bytes=319029864 weight_movement_bytes=25165824 total_nvlink_bytes=344195688 weight_to_token_ratio=0.078882 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404332180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44526 payload_rows_remote=38770 dup_rows=21010 dup_rows_remote=18210 dedup_ratio=0.320587 router_tokens_per_expert=[569, 535, 471, 514, 495, 538, 487, 519, 506, 512, 523, 514, 520, 477, 512, 459, 522, 527, 518, 504, 523, 507, 484, 491, 511, 529, 535, 518, 514, 498, 531, 505, 517, 515, 503, 495, 537, 469, 496, 450, 524, 478, 488, 499, 532, 520, 534, 515, 524, 518, 554, 511, 527, 519, 569, 505, 513, 516, 513, 520, 547, 487, 498, 507] payload_rows_per_dst=[5756, 5584, 5466, 5586, 5402, 5680, 5412, 5640] dup_rows_per_dst=[2800, 2688, 2522, 2650, 2562, 2876, 2350, 2562] combine_rows_per_src=[5756, 5584, 5466, 5586, 5402, 5680, 5412, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159801344 dispatch_meta_bytes=228784 combine_payload_bytes=159801344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160030128 ingress_bytes=184967168 link_bytes=184967168 token_bytes=319831472 weight_movement_bytes=25165824 total_nvlink_bytes=344997296 weight_to_token_ratio=0.078685 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407386431e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159096832 dispatch_meta_bytes=229400 combine_payload_bytes=159096832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159326232 ingress_bytes=209428480 link_bytes=209428480 token_bytes=318423064 weight_movement_bytes=50331648 total_nvlink_bytes=368754712 weight_to_token_ratio=0.158065 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593508752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=229464 combine_payload_bytes=159285248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159514712 ingress_bytes=184451072 link_bytes=184451072 token_bytes=318799960 weight_movement_bytes=25165824 total_nvlink_bytes=343965784 weight_to_token_ratio=0.078939 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403459536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158859264 dispatch_meta_bytes=229360 combine_payload_bytes=158859264 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159088624 ingress_bytes=234356736 link_bytes=234356736 token_bytes=317947888 weight_movement_bytes=75497472 total_nvlink_bytes=393445360 weight_to_token_ratio=0.237452 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783183977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44426 payload_rows_remote=38888 dup_rows=21110 dup_rows_remote=18506 dedup_ratio=0.322113 router_tokens_per_expert=[518, 510, 530, 498, 512, 530, 521, 504, 504, 507, 520, 531, 537, 529, 515, 489, 529, 528, 515, 508, 491, 502, 514, 498, 497, 483, 518, 536, 494, 509, 470, 500, 478, 498, 509, 489, 518, 530, 510, 528, 561, 515, 528, 522, 497, 502, 496, 507, 506, 472, 511, 536, 499, 516, 501, 530, 539, 566, 510, 514, 507, 501, 519, 506] payload_rows_per_dst=[5570, 5528, 5570, 5530, 5472, 5640, 5538, 5578] dup_rows_per_dst=[2676, 2736, 2600, 2484, 2648, 2616, 2604, 2746] combine_rows_per_src=[5570, 5528, 5570, 5530, 5472, 5640, 5538, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158801920 dispatch_meta_bytes=227920 combine_payload_bytes=158801920 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159029840 ingress_bytes=183967744 link_bytes=183967744 token_bytes=317831760 weight_movement_bytes=25165824 total_nvlink_bytes=342997584 weight_to_token_ratio=0.079180 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399781969e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38928 dup_rows=21100 dup_rows_remote=18430 dedup_ratio=0.321960 router_tokens_per_expert=[520, 507, 518, 512, 520, 552, 503, 509, 509, 517, 512, 533, 498, 506, 534, 502, 533, 493, 526, 515, 500, 509, 525, 514, 496, 499, 514, 498, 506, 520, 519, 526, 500, 521, 515, 492, 518, 504, 512, 527, 482, 493, 520, 511, 545, 472, 547, 532, 475, 480, 480, 540, 509, 544, 526, 530, 527, 509, 502, 480, 526, 509, 518, 477] payload_rows_per_dst=[5618, 5580, 5532, 5598, 5508, 5508, 5580, 5512] dup_rows_per_dst=[2664, 2642, 2698, 2558, 2670, 2696, 2588, 2584] combine_rows_per_src=[5618, 5580, 5532, 5598, 5508, 5508, 5580, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44614 payload_rows_remote=39034 dup_rows=20922 dup_rows_remote=18446 dedup_ratio=0.319244 router_tokens_per_expert=[479, 524, 541, 536, 489, 525, 541, 521, 538, 489, 507, 556, 518, 523, 500, 510, 478, 497, 513, 535, 538, 540, 527, 499, 491, 468, 518, 537, 493, 496, 529, 490, 478, 508, 493, 510, 490, 523, 512, 521, 508, 518, 534, 466, 468, 526, 501, 507, 483, 519, 545, 477, 479, 516, 507, 538, 484, 537, 528, 561, 490, 536, 550, 509] payload_rows_per_dst=[5654, 5644, 5556, 5492, 5540, 5580, 5530, 5618] dup_rows_per_dst=[2658, 2638, 2698, 2552, 2530, 2476, 2598, 2772] combine_rows_per_src=[5654, 5644, 5556, 5492, 5540, 5580, 5530, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44324 payload_rows_remote=38764 dup_rows=21212 dup_rows_remote=18526 dedup_ratio=0.323669 router_tokens_per_expert=[537, 473, 536, 505, 513, 512, 483, 500, 526, 522, 535, 545, 505, 513, 495, 538, 518, 505, 509, 504, 521, 523, 481, 553, 494, 508, 518, 482, 462, 487, 500, 506, 496, 531, 518, 523, 516, 530, 528, 487, 513, 506, 513, 494, 527, 491, 533, 536, 508, 523, 521, 491, 507, 539, 520, 541, 472, 542, 480, 551, 490, 524, 501, 507] payload_rows_per_dst=[5404, 5362, 5246, 5642, 5856, 5632, 5622, 5560] dup_rows_per_dst=[2470, 2402, 2382, 2776, 3056, 2714, 2726, 2686] combine_rows_per_src=[5404, 5362, 5246, 5642, 5856, 5632, 5622, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=229576 combine_payload_bytes=159285248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159514824 ingress_bytes=184451072 link_bytes=184451072 token_bytes=318800072 weight_movement_bytes=25165824 total_nvlink_bytes=343965896 weight_to_token_ratio=0.078939 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403459536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38864 dup_rows=21072 dup_rows_remote=18500 dedup_ratio=0.321533 router_tokens_per_expert=[515, 525, 511, 535, 518, 487, 526, 511, 558, 513, 507, 512, 536, 466, 501, 493, 510, 487, 521, 510, 535, 527, 556, 508, 496, 531, 532, 514, 538, 538, 519, 541, 453, 517, 510, 503, 494, 494, 469, 521, 499, 500, 522, 483, 493, 496, 542, 475, 518, 519, 525, 549, 529, 525, 515, 478, 506, 514, 516, 472, 513, 504, 471, 566] payload_rows_per_dst=[5608, 5600, 5594, 5596, 5494, 5478, 5588, 5506] dup_rows_per_dst=[2648, 2572, 2714, 2822, 2428, 2542, 2728, 2618] combine_rows_per_src=[5608, 5600, 5594, 5596, 5494, 5478, 5588, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38808 dup_rows=21046 dup_rows_remote=18322 dedup_ratio=0.321136 router_tokens_per_expert=[532, 497, 543, 512, 460, 511, 536, 510, 523, 536, 486, 514, 501, 515, 514, 515, 546, 533, 550, 532, 495, 517, 515, 515, 530, 515, 485, 509, 490, 531, 500, 503, 476, 507, 527, 513, 513, 558, 510, 508, 538, 504, 518, 536, 501, 501, 494, 512, 491, 522, 545, 479, 488, 518, 509, 518, 485, 514, 501, 494, 491, 529, 467, 530] payload_rows_per_dst=[5542, 5556, 5682, 5576, 5608, 5482, 5558, 5486] dup_rows_per_dst=[2660, 2652, 2724, 2550, 2616, 2726, 2582, 2536] combine_rows_per_src=[5542, 5556, 5682, 5576, 5608, 5482, 5558, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38886 dup_rows=21098 dup_rows_remote=18468 dedup_ratio=0.321930 router_tokens_per_expert=[490, 500, 518, 508, 503, 515, 499, 529, 540, 537, 500, 541, 541, 529, 536, 518, 513, 495, 521, 514, 513, 581, 497, 518, 532, 501, 486, 495, 502, 537, 513, 525, 479, 480, 499, 517, 532, 509, 498, 485, 495, 514, 549, 463, 472, 535, 503, 487, 490, 536, 535, 479, 510, 524, 481, 516, 485, 504, 520, 526, 526, 487, 559, 526] payload_rows_per_dst=[5548, 5736, 5554, 5552, 5480, 5450, 5570, 5548] dup_rows_per_dst=[2576, 2748, 2750, 2630, 2518, 2586, 2572, 2718] combine_rows_per_src=[5548, 5736, 5554, 5552, 5480, 5450, 5570, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159449088 dispatch_meta_bytes=229432 combine_payload_bytes=159449088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159678520 ingress_bytes=184614912 link_bytes=184614912 token_bytes=319127608 weight_movement_bytes=25165824 total_nvlink_bytes=344293432 weight_to_token_ratio=0.078858 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404706170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=229920 combine_payload_bytes=159883264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160113184 ingress_bytes=185049088 link_bytes=185049088 token_bytes=319996448 weight_movement_bytes=25165824 total_nvlink_bytes=345162272 weight_to_token_ratio=0.078644 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408009747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158777344 dispatch_meta_bytes=229160 combine_payload_bytes=158777344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159006504 ingress_bytes=183943168 link_bytes=183943168 token_bytes=317783848 weight_movement_bytes=25165824 total_nvlink_bytes=342949672 weight_to_token_ratio=0.079192 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399594974e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=39042 dup_rows=20910 dup_rows_remote=18254 dedup_ratio=0.319061 router_tokens_per_expert=[521, 454, 529, 509, 503, 525, 556, 523, 492, 521, 561, 486, 496, 508, 517, 489, 499, 547, 527, 521, 514, 522, 473, 515, 521, 527, 471, 513, 543, 531, 489, 494, 510, 485, 489, 545, 503, 525, 497, 502, 501, 513, 527, 562, 492, 574, 525, 511, 510, 495, 518, 503, 529, 489, 511, 521, 505, 506, 536, 528, 465, 501, 516, 477] payload_rows_per_dst=[5584, 5530, 5572, 5600, 5508, 5740, 5568, 5524] dup_rows_per_dst=[2656, 2610, 2664, 2578, 2604, 2670, 2584, 2544] combine_rows_per_src=[5584, 5530, 5572, 5600, 5508, 5740, 5568, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159186944 dispatch_meta_bytes=229456 combine_payload_bytes=159186944 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159416400 ingress_bytes=259850240 link_bytes=259850240 token_bytes=318603344 weight_movement_bytes=100663296 total_nvlink_bytes=419266640 weight_to_token_ratio=0.315952 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.977160086e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158957568 dispatch_meta_bytes=228520 combine_payload_bytes=158957568 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159186088 ingress_bytes=209289216 link_bytes=209289216 token_bytes=318143656 weight_movement_bytes=50331648 total_nvlink_bytes=368475304 weight_to_token_ratio=0.158204 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592449113e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=229416 combine_payload_bytes=159277056 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159506472 ingress_bytes=184442880 link_bytes=184442880 token_bytes=318783528 weight_movement_bytes=25165824 total_nvlink_bytes=343949352 weight_to_token_ratio=0.078943 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403397205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159916032 dispatch_meta_bytes=229184 combine_payload_bytes=159916032 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160145216 ingress_bytes=235413504 link_bytes=235413504 token_bytes=320061248 weight_movement_bytes=75497472 total_nvlink_bytes=395558720 weight_to_token_ratio=0.235884 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791224760e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44600 payload_rows_remote=39092 dup_rows=20936 dup_rows_remote=18430 dedup_ratio=0.319458 router_tokens_per_expert=[485, 507, 506, 524, 507, 515, 506, 503, 498, 577, 495, 491, 502, 527, 551, 532, 498, 520, 525, 505, 519, 486, 537, 511, 525, 529, 502, 506, 493, 525, 491, 505, 519, 509, 524, 461, 564, 530, 492, 474, 506, 524, 518, 530, 525, 526, 525, 535, 494, 492, 494, 523, 513, 512, 473, 506, 540, 505, 504, 504, 514, 502, 503, 524] payload_rows_per_dst=[5548, 5620, 5588, 5564, 5540, 5654, 5508, 5578] dup_rows_per_dst=[2558, 2726, 2614, 2588, 2606, 2724, 2506, 2614] combine_rows_per_src=[5548, 5620, 5588, 5564, 5540, 5654, 5508, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=38940 dup_rows=20946 dup_rows_remote=18254 dedup_ratio=0.319611 router_tokens_per_expert=[517, 522, 545, 535, 518, 487, 522, 524, 507, 538, 477, 498, 463, 508, 539, 484, 514, 561, 545, 537, 499, 523, 506, 480, 511, 477, 481, 494, 514, 466, 555, 501, 533, 499, 517, 508, 489, 500, 484, 485, 563, 513, 508, 493, 498, 488, 558, 550, 494, 490, 523, 531, 530, 525, 566, 508, 492, 502, 479, 501, 513, 519, 509, 552] payload_rows_per_dst=[5648, 5446, 5694, 5478, 5490, 5650, 5618, 5566] dup_rows_per_dst=[2692, 2582, 2636, 2520, 2540, 2692, 2716, 2568] combine_rows_per_src=[5648, 5446, 5694, 5478, 5490, 5650, 5618, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44598 payload_rows_remote=38852 dup_rows=20938 dup_rows_remote=18254 dedup_ratio=0.319489 router_tokens_per_expert=[500, 542, 502, 518, 503, 539, 507, 529, 527, 517, 527, 534, 497, 532, 532, 510, 518, 465, 494, 509, 497, 471, 515, 509, 517, 481, 527, 533, 506, 545, 525, 510, 523, 524, 529, 535, 511, 497, 515, 520, 507, 495, 514, 511, 531, 528, 500, 496, 516, 508, 547, 487, 501, 486, 510, 484, 509, 516, 502, 516, 527, 508, 490, 487] payload_rows_per_dst=[5526, 5660, 5540, 5576, 5608, 5534, 5408, 5746] dup_rows_per_dst=[2554, 2872, 2416, 2712, 2714, 2598, 2388, 2684] combine_rows_per_src=[5526, 5660, 5540, 5576, 5608, 5534, 5408, 5746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=39068 dup_rows=20910 dup_rows_remote=18468 dedup_ratio=0.319061 router_tokens_per_expert=[515, 521, 509, 516, 514, 514, 506, 522, 502, 500, 527, 512, 530, 517, 501, 479, 500, 529, 486, 487, 482, 486, 475, 509, 536, 520, 546, 527, 472, 537, 497, 507, 491, 480, 519, 504, 524, 489, 490, 503, 539, 497, 489, 523, 531, 511, 526, 521, 513, 562, 515, 532, 538, 539, 523, 507, 543, 512, 488, 473, 522, 519, 531, 533] payload_rows_per_dst=[5580, 5568, 5518, 5550, 5558, 5632, 5674, 5546] dup_rows_per_dst=[2654, 2568, 2390, 2734, 2442, 2642, 2784, 2696] combine_rows_per_src=[5580, 5568, 5518, 5550, 5558, 5632, 5674, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160120832 dispatch_meta_bytes=230088 combine_payload_bytes=160120832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160350920 ingress_bytes=210452480 link_bytes=210452480 token_bytes=320471752 weight_movement_bytes=50331648 total_nvlink_bytes=370803400 weight_to_token_ratio=0.157055 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601300208e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44320 payload_rows_remote=38786 dup_rows=21216 dup_rows_remote=18610 dedup_ratio=0.323730 router_tokens_per_expert=[515, 493, 510, 498, 507, 540, 477, 519, 514, 497, 509, 508, 490, 526, 524, 502, 505, 503, 525, 499, 513, 525, 515, 523, 513, 495, 523, 494, 541, 536, 495, 526, 509, 476, 521, 538, 529, 503, 482, 547, 565, 490, 519, 554, 514, 506, 514, 511, 480, 512, 507, 526, 486, 516, 514, 532, 556, 483, 516, 469, 520, 475, 518, 520] payload_rows_per_dst=[5512, 5534, 5580, 5548, 5498, 5582, 5572, 5494] dup_rows_per_dst=[2606, 2606, 2636, 2698, 2712, 2764, 2574, 2620] combine_rows_per_src=[5512, 5534, 5580, 5548, 5498, 5582, 5572, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44430 payload_rows_remote=38874 dup_rows=21106 dup_rows_remote=18560 dedup_ratio=0.322052 router_tokens_per_expert=[554, 537, 503, 552, 510, 501, 523, 518, 504, 527, 480, 500, 479, 506, 518, 513, 526, 440, 535, 492, 489, 518, 529, 522, 521, 514, 510, 525, 530, 497, 526, 524, 490, 514, 513, 554, 494, 500, 500, 519, 496, 519, 466, 510, 518, 558, 507, 531, 480, 541, 533, 558, 475, 524, 520, 521, 513, 521, 485, 493, 503, 496, 486, 507] payload_rows_per_dst=[5766, 5456, 5556, 5544, 5504, 5578, 5520, 5506] dup_rows_per_dst=[2630, 2598, 2546, 2750, 2664, 2632, 2784, 2502] combine_rows_per_src=[5766, 5456, 5556, 5544, 5504, 5578, 5520, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=39002 dup_rows=21036 dup_rows_remote=18506 dedup_ratio=0.320984 router_tokens_per_expert=[510, 490, 502, 528, 520, 481, 541, 499, 517, 522, 537, 497, 513, 521, 514, 508, 495, 491, 510, 491, 508, 497, 496, 491, 500, 446, 480, 514, 527, 545, 490, 512, 568, 506, 547, 537, 460, 535, 518, 529, 511, 498, 505, 529, 501, 523, 563, 521, 516, 511, 529, 509, 520, 487, 529, 493, 545, 527, 523, 501, 513, 486, 519, 516] payload_rows_per_dst=[5508, 5548, 5518, 5498, 5660, 5624, 5536, 5608] dup_rows_per_dst=[2634, 2710, 2440, 2530, 2740, 2678, 2652, 2652] combine_rows_per_src=[5508, 5548, 5518, 5498, 5660, 5624, 5536, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=228776 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159727016 ingress_bytes=234995712 link_bytes=234995712 token_bytes=319225256 weight_movement_bytes=75497472 total_nvlink_bytes=394722728 weight_to_token_ratio=0.236502 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788045846e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159137792 dispatch_meta_bytes=228424 combine_payload_bytes=159137792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159366216 ingress_bytes=184303616 link_bytes=184303616 token_bytes=318504008 weight_movement_bytes=25165824 total_nvlink_bytes=343669832 weight_to_token_ratio=0.079013 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402337567e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160022528 dispatch_meta_bytes=230144 combine_payload_bytes=160022528 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160252672 ingress_bytes=210354176 link_bytes=210354176 token_bytes=320275200 weight_movement_bytes=50331648 total_nvlink_bytes=370606848 weight_to_token_ratio=0.157151 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600552229e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44572 payload_rows_remote=39010 dup_rows=20964 dup_rows_remote=18406 dedup_ratio=0.319885 router_tokens_per_expert=[528, 508, 514, 507, 513, 499, 509, 482, 526, 509, 500, 493, 512, 530, 495, 456, 494, 493, 532, 525, 510, 482, 485, 550, 471, 509, 496, 525, 499, 530, 500, 521, 511, 533, 484, 548, 573, 502, 551, 500, 540, 529, 509, 537, 531, 483, 462, 546, 510, 509, 495, 546, 499, 553, 525, 511, 530, 496, 527, 496, 480, 527, 517, 505] payload_rows_per_dst=[5562, 5514, 5792, 5630, 5570, 5372, 5578, 5554] dup_rows_per_dst=[2558, 2528, 3072, 2616, 2690, 2180, 2718, 2602] combine_rows_per_src=[5562, 5514, 5792, 5630, 5570, 5372, 5578, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=38988 dup_rows=20990 dup_rows_remote=18216 dedup_ratio=0.320282 router_tokens_per_expert=[512, 506, 531, 524, 521, 496, 560, 513, 498, 501, 456, 488, 516, 529, 483, 517, 476, 513, 561, 573, 568, 468, 527, 519, 521, 484, 529, 466, 501, 540, 517, 504, 517, 507, 548, 500, 540, 482, 501, 499, 513, 496, 483, 502, 466, 542, 519, 513, 497, 531, 525, 509, 521, 535, 511, 537, 499, 502, 490, 521, 542, 510, 494, 498] payload_rows_per_dst=[5644, 5494, 5638, 5566, 5580, 5490, 5558, 5576] dup_rows_per_dst=[2682, 2482, 2772, 2558, 2608, 2578, 2774, 2536] combine_rows_per_src=[5644, 5494, 5638, 5566, 5580, 5490, 5558, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158867456 dispatch_meta_bytes=229584 combine_payload_bytes=158867456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159097040 ingress_bytes=184033280 link_bytes=184033280 token_bytes=317964496 weight_movement_bytes=25165824 total_nvlink_bytes=343130320 weight_to_token_ratio=0.079147 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400280622e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159227904 dispatch_meta_bytes=229736 combine_payload_bytes=159227904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159457640 ingress_bytes=184393728 link_bytes=184393728 token_bytes=318685544 weight_movement_bytes=25165824 total_nvlink_bytes=343851368 weight_to_token_ratio=0.078968 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403023215e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159752192 dispatch_meta_bytes=230032 combine_payload_bytes=159752192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159982224 ingress_bytes=184918016 link_bytes=184918016 token_bytes=319734416 weight_movement_bytes=25165824 total_nvlink_bytes=344900240 weight_to_token_ratio=0.078709 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407012441e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=229664 combine_payload_bytes=159784960 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160014624 ingress_bytes=235282432 link_bytes=235282432 token_bytes=319799584 weight_movement_bytes=75497472 total_nvlink_bytes=395297056 weight_to_token_ratio=0.236077 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790227454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=228816 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159923664 ingress_bytes=210026496 link_bytes=210026496 token_bytes=319618512 weight_movement_bytes=50331648 total_nvlink_bytes=369950160 weight_to_token_ratio=0.157474 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598058962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44458 payload_rows_remote=38998 dup_rows=21078 dup_rows_remote=18600 dedup_ratio=0.321625 router_tokens_per_expert=[518, 533, 476, 505, 478, 553, 550, 533, 548, 542, 517, 528, 526, 511, 510, 503, 496, 531, 480, 533, 504, 490, 518, 524, 513, 476, 497, 479, 498, 485, 528, 498, 526, 521, 515, 523, 563, 494, 518, 486, 513, 467, 512, 507, 479, 515, 496, 480, 524, 514, 541, 544, 525, 498, 550, 506, 531, 495, 489, 508, 534, 487, 505, 521] payload_rows_per_dst=[5624, 5614, 5498, 5416, 5694, 5460, 5630, 5522] dup_rows_per_dst=[2668, 2756, 2654, 2532, 2598, 2478, 2774, 2618] combine_rows_per_src=[5624, 5614, 5498, 5416, 5694, 5460, 5630, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38790 dup_rows=21088 dup_rows_remote=18144 dedup_ratio=0.321777 router_tokens_per_expert=[506, 483, 484, 496, 481, 524, 509, 523, 486, 516, 484, 516, 528, 557, 502, 517, 525, 508, 528, 503, 525, 545, 481, 518, 523, 505, 489, 499, 499, 557, 514, 508, 521, 527, 541, 475, 492, 462, 508, 502, 519, 518, 511, 496, 524, 498, 483, 517, 552, 529, 543, 500, 492, 532, 504, 541, 525, 511, 513, 496, 546, 485, 528, 538] payload_rows_per_dst=[5244, 5444, 5618, 5646, 5540, 5474, 5824, 5658] dup_rows_per_dst=[2124, 2450, 2648, 2728, 2516, 2658, 3020, 2944] combine_rows_per_src=[5244, 5444, 5618, 5646, 5540, 5474, 5824, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38846 dup_rows=21108 dup_rows_remote=18364 dedup_ratio=0.322083 router_tokens_per_expert=[513, 530, 472, 504, 549, 545, 521, 488, 551, 540, 529, 531, 448, 528, 510, 575, 497, 493, 507, 501, 483, 532, 497, 481, 542, 532, 508, 503, 526, 487, 506, 499, 513, 514, 532, 530, 521, 508, 524, 521, 498, 508, 520, 483, 513, 495, 516, 503, 500, 497, 535, 526, 523, 527, 485, 509, 514, 494, 511, 476, 529, 513, 527, 475] payload_rows_per_dst=[5582, 5690, 5482, 5580, 5582, 5444, 5592, 5476] dup_rows_per_dst=[2662, 2734, 2500, 2626, 2744, 2628, 2612, 2602] combine_rows_per_src=[5582, 5690, 5482, 5580, 5582, 5444, 5592, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159735808 dispatch_meta_bytes=230392 combine_payload_bytes=159735808 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159966200 ingress_bytes=184901632 link_bytes=184901632 token_bytes=319702008 weight_movement_bytes=25165824 total_nvlink_bytes=344867832 weight_to_token_ratio=0.078717 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406887777e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44646 payload_rows_remote=39078 dup_rows=20890 dup_rows_remote=18308 dedup_ratio=0.318756 router_tokens_per_expert=[543, 538, 529, 532, 508, 522, 520, 524, 501, 535, 496, 482, 513, 492, 489, 567, 524, 514, 519, 520, 498, 487, 522, 477, 557, 509, 511, 497, 501, 526, 532, 501, 492, 510, 510, 501, 501, 534, 518, 501, 455, 530, 527, 523, 502, 492, 493, 496, 532, 490, 540, 534, 551, 520, 506, 490, 449, 487, 535, 524, 504, 535, 520, 480] payload_rows_per_dst=[5720, 5568, 5558, 5592, 5554, 5538, 5554, 5562] dup_rows_per_dst=[2712, 2582, 2564, 2676, 2580, 2618, 2652, 2506] combine_rows_per_src=[5720, 5568, 5558, 5592, 5554, 5538, 5554, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158883840 dispatch_meta_bytes=227736 combine_payload_bytes=158883840 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159111576 ingress_bytes=184049664 link_bytes=184049664 token_bytes=317995416 weight_movement_bytes=25165824 total_nvlink_bytes=343161240 weight_to_token_ratio=0.079139 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400405285e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44696 payload_rows_remote=39052 dup_rows=20840 dup_rows_remote=18114 dedup_ratio=0.317993 router_tokens_per_expert=[544, 540, 517, 509, 507, 514, 553, 480, 551, 534, 480, 473, 522, 502, 511, 508, 525, 541, 502, 558, 529, 501, 506, 518, 492, 501, 547, 517, 512, 479, 515, 494, 506, 467, 525, 511, 520, 512, 539, 523, 462, 458, 508, 514, 493, 484, 501, 491, 477, 494, 509, 495, 533, 524, 550, 543, 528, 501, 490, 523, 525, 505, 525, 550] payload_rows_per_dst=[5644, 5582, 5640, 5572, 5624, 5806, 5220, 5608] dup_rows_per_dst=[2726, 2594, 2712, 2542, 2534, 3034, 2012, 2686] combine_rows_per_src=[5644, 5582, 5640, 5572, 5624, 5806, 5220, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44638 payload_rows_remote=39038 dup_rows=20898 dup_rows_remote=18368 dedup_ratio=0.318878 router_tokens_per_expert=[565, 504, 500, 507, 508, 531, 485, 494, 520, 516, 513, 527, 522, 482, 571, 541, 494, 531, 510, 471, 511, 496, 501, 551, 520, 536, 517, 529, 538, 546, 494, 535, 490, 475, 500, 548, 494, 495, 516, 486, 500, 524, 509, 484, 496, 556, 529, 514, 527, 512, 534, 492, 494, 510, 538, 472, 510, 530, 502, 487, 524, 475, 483, 496] payload_rows_per_dst=[5548, 5626, 5600, 5690, 5496, 5642, 5550, 5486] dup_rows_per_dst=[2640, 2758, 2530, 2740, 2512, 2582, 2608, 2528] combine_rows_per_src=[5548, 5626, 5600, 5690, 5496, 5642, 5550, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=228840 combine_payload_bytes=159113216 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159342056 ingress_bytes=209444864 link_bytes=209444864 token_bytes=318455272 weight_movement_bytes=50331648 total_nvlink_bytes=368786920 weight_to_token_ratio=0.158049 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593633415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38942 dup_rows=21036 dup_rows_remote=18472 dedup_ratio=0.320984 router_tokens_per_expert=[515, 513, 539, 523, 530, 539, 509, 506, 495, 494, 518, 490, 499, 490, 533, 516, 518, 510, 520, 523, 472, 501, 538, 511, 536, 546, 515, 504, 520, 532, 496, 527, 482, 493, 518, 512, 500, 523, 527, 506, 543, 523, 501, 515, 496, 542, 509, 524, 530, 488, 504, 544, 480, 475, 510, 530, 538, 505, 483, 497, 498, 484, 491, 519] payload_rows_per_dst=[5576, 5510, 5586, 5642, 5528, 5590, 5558, 5510] dup_rows_per_dst=[2772, 2560, 2600, 2710, 2594, 2716, 2564, 2520] combine_rows_per_src=[5576, 5510, 5586, 5642, 5528, 5590, 5558, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44668 payload_rows_remote=39090 dup_rows=20868 dup_rows_remote=18374 dedup_ratio=0.318420 router_tokens_per_expert=[508, 509, 528, 534, 502, 525, 522, 530, 476, 516, 515, 508, 520, 495, 527, 524, 498, 515, 507, 490, 507, 491, 538, 515, 495, 499, 508, 509, 513, 492, 516, 504, 534, 498, 518, 545, 559, 535, 480, 518, 534, 493, 499, 529, 512, 482, 505, 499, 538, 487, 489, 507, 535, 487, 526, 509, 505, 497, 551, 517, 504, 554, 490, 496] payload_rows_per_dst=[5604, 5596, 5544, 5578, 5686, 5518, 5582, 5560] dup_rows_per_dst=[2712, 2566, 2578, 2494, 2688, 2588, 2574, 2668] combine_rows_per_src=[5604, 5596, 5544, 5578, 5686, 5518, 5582, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=229544 combine_payload_bytes=160063488 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160293032 ingress_bytes=210395136 link_bytes=210395136 token_bytes=320356520 weight_movement_bytes=50331648 total_nvlink_bytes=370688168 weight_to_token_ratio=0.157111 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600863887e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159956992 dispatch_meta_bytes=228664 combine_payload_bytes=159956992 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160185656 ingress_bytes=235454464 link_bytes=235454464 token_bytes=320142648 weight_movement_bytes=75497472 total_nvlink_bytes=395640120 weight_to_token_ratio=0.235824 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791536418e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159899648 dispatch_meta_bytes=229624 combine_payload_bytes=159899648 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160129272 ingress_bytes=210231296 link_bytes=210231296 token_bytes=320028920 weight_movement_bytes=50331648 total_nvlink_bytes=370360568 weight_to_token_ratio=0.157272 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599617254e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=229656 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159736088 ingress_bytes=184672256 link_bytes=184672256 token_bytes=319242520 weight_movement_bytes=25165824 total_nvlink_bytes=344408344 weight_to_token_ratio=0.078830 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405142491e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160112640 dispatch_meta_bytes=229856 combine_payload_bytes=160112640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160342496 ingress_bytes=185278464 link_bytes=185278464 token_bytes=320455136 weight_movement_bytes=25165824 total_nvlink_bytes=345620960 weight_to_token_ratio=0.078532 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409755034e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44612 payload_rows_remote=38966 dup_rows=20924 dup_rows_remote=18186 dedup_ratio=0.319275 router_tokens_per_expert=[501, 489, 478, 493, 558, 505, 539, 505, 535, 520, 527, 507, 543, 526, 505, 505, 500, 514, 515, 475, 518, 490, 507, 506, 482, 513, 463, 503, 536, 517, 516, 521, 536, 538, 500, 496, 516, 521, 512, 495, 511, 536, 482, 556, 542, 533, 531, 501, 513, 498, 506, 517, 492, 486, 524, 523, 500, 503, 534, 512, 490, 529, 505, 518] payload_rows_per_dst=[5580, 5624, 5488, 5554, 5576, 5646, 5540, 5604] dup_rows_per_dst=[2556, 2712, 2562, 2548, 2652, 2738, 2578, 2578] combine_rows_per_src=[5580, 5624, 5488, 5554, 5576, 5646, 5540, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=38820 dup_rows=20978 dup_rows_remote=18176 dedup_ratio=0.320099 router_tokens_per_expert=[533, 500, 522, 511, 515, 493, 541, 522, 498, 507, 525, 516, 525, 491, 494, 502, 529, 538, 506, 521, 505, 539, 531, 512, 557, 507, 532, 557, 509, 509, 508, 506, 503, 495, 498, 473, 513, 498, 523, 487, 454, 514, 518, 496, 498, 538, 478, 497, 509, 481, 538, 528, 534, 539, 478, 499, 556, 527, 474, 508, 490, 508, 504, 551] payload_rows_per_dst=[5610, 5556, 5352, 5616, 5622, 5370, 5694, 5738] dup_rows_per_dst=[2664, 2560, 2362, 2754, 2656, 2312, 2868, 2802] combine_rows_per_src=[5610, 5556, 5352, 5616, 5622, 5370, 5694, 5738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38776 dup_rows=21142 dup_rows_remote=18482 dedup_ratio=0.322601 router_tokens_per_expert=[488, 475, 532, 490, 468, 508, 498, 539, 514, 507, 465, 527, 545, 514, 469, 521, 513, 532, 517, 532, 512, 537, 520, 480, 538, 524, 520, 460, 538, 514, 505, 513, 533, 512, 533, 530, 550, 476, 467, 538, 544, 507, 541, 539, 499, 512, 521, 526, 533, 518, 491, 536, 523, 490, 507, 541, 504, 503, 462, 494, 524, 476, 498, 525] payload_rows_per_dst=[5498, 5532, 5556, 5496, 5618, 5590, 5584, 5520] dup_rows_per_dst=[2498, 2592, 2730, 2728, 2660, 2788, 2694, 2452] combine_rows_per_src=[5498, 5532, 5556, 5496, 5618, 5590, 5584, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=227984 combine_payload_bytes=159006720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159234704 ingress_bytes=184172544 link_bytes=184172544 token_bytes=318241424 weight_movement_bytes=25165824 total_nvlink_bytes=343407248 weight_to_token_ratio=0.079078 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401340260e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159604736 dispatch_meta_bytes=228608 combine_payload_bytes=159604736 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159833344 ingress_bytes=235102208 link_bytes=235102208 token_bytes=319438080 weight_movement_bytes=75497472 total_nvlink_bytes=394935552 weight_to_token_ratio=0.236345 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788856157e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=38930 dup_rows=20982 dup_rows_remote=18386 dedup_ratio=0.320160 router_tokens_per_expert=[513, 513, 504, 487, 490, 544, 513, 527, 510, 509, 532, 497, 513, 484, 515, 539, 536, 516, 532, 525, 510, 513, 523, 544, 489, 525, 511, 505, 514, 505, 475, 503, 527, 492, 512, 509, 536, 484, 525, 510, 496, 519, 510, 503, 532, 531, 517, 516, 543, 514, 518, 472, 490, 526, 515, 456, 525, 504, 512, 536, 504, 503, 507, 508] payload_rows_per_dst=[5624, 5666, 5736, 5316, 5540, 5586, 5494, 5592] dup_rows_per_dst=[2596, 2848, 2802, 2296, 2650, 2610, 2574, 2606] combine_rows_per_src=[5624, 5666, 5736, 5316, 5540, 5586, 5494, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=229032 combine_payload_bytes=158826496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159055528 ingress_bytes=183992320 link_bytes=183992320 token_bytes=317882024 weight_movement_bytes=25165824 total_nvlink_bytes=343047848 weight_to_token_ratio=0.079167 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399968964e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44276 payload_rows_remote=38782 dup_rows=21260 dup_rows_remote=18566 dedup_ratio=0.324402 router_tokens_per_expert=[519, 469, 510, 552, 532, 518, 542, 525, 513, 522, 532, 515, 510, 500, 480, 522, 537, 508, 489, 525, 547, 518, 518, 530, 511, 506, 500, 488, 502, 530, 514, 542, 513, 479, 510, 499, 507, 476, 489, 503, 475, 500, 504, 515, 561, 535, 442, 497, 514, 476, 536, 518, 495, 533, 486, 538, 539, 503, 498, 493, 502, 528, 553, 525] payload_rows_per_dst=[5670, 5494, 5576, 5518, 5418, 5464, 5562, 5574] dup_rows_per_dst=[2664, 2694, 2768, 2668, 2534, 2594, 2630, 2708] combine_rows_per_src=[5670, 5494, 5576, 5518, 5418, 5464, 5562, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44374 payload_rows_remote=38784 dup_rows=21162 dup_rows_remote=18512 dedup_ratio=0.322906 router_tokens_per_expert=[546, 470, 494, 492, 524, 481, 476, 538, 506, 490, 498, 499, 512, 532, 537, 513, 522, 514, 502, 519, 521, 519, 522, 501, 504, 505, 512, 516, 526, 513, 565, 496, 495, 528, 501, 504, 517, 531, 506, 512, 557, 536, 488, 524, 510, 524, 501, 514, 554, 502, 493, 484, 474, 523, 487, 502, 495, 532, 540, 520, 502, 513, 489, 545] payload_rows_per_dst=[5398, 5518, 5590, 5594, 5618, 5612, 5534, 5510] dup_rows_per_dst=[2644, 2656, 2650, 2680, 2570, 2696, 2504, 2762] combine_rows_per_src=[5398, 5518, 5590, 5594, 5618, 5612, 5534, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38898 dup_rows=21068 dup_rows_remote=18348 dedup_ratio=0.321472 router_tokens_per_expert=[505, 511, 512, 503, 444, 539, 516, 495, 509, 500, 535, 500, 486, 516, 493, 504, 503, 532, 444, 546, 530, 549, 513, 487, 528, 543, 513, 515, 503, 520, 484, 487, 522, 510, 489, 569, 516, 488, 515, 514, 494, 505, 533, 532, 500, 505, 528, 533, 492, 506, 530, 561, 524, 498, 521, 513, 520, 495, 547, 508, 497, 537, 508, 493] payload_rows_per_dst=[5530, 5476, 5592, 5570, 5586, 5566, 5570, 5578] dup_rows_per_dst=[2520, 2610, 2616, 2616, 2660, 2694, 2720, 2632] combine_rows_per_src=[5530, 5476, 5592, 5570, 5586, 5566, 5570, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44644 payload_rows_remote=38944 dup_rows=20892 dup_rows_remote=18176 dedup_ratio=0.318787 router_tokens_per_expert=[523, 525, 493, 480, 535, 512, 519, 506, 496, 516, 497, 513, 508, 484, 490, 503, 514, 506, 508, 509, 501, 516, 518, 517, 529, 524, 533, 530, 529, 515, 512, 536, 530, 528, 474, 519, 546, 489, 533, 531, 487, 507, 527, 497, 530, 520, 533, 511, 477, 527, 499, 545, 518, 507, 495, 511, 521, 491, 531, 525, 488, 476, 490, 508] payload_rows_per_dst=[5498, 5536, 5576, 5700, 5628, 5572, 5602, 5532] dup_rows_per_dst=[2688, 2478, 2602, 2716, 2672, 2652, 2556, 2528] combine_rows_per_src=[5498, 5536, 5576, 5700, 5628, 5572, 5602, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159457280 dispatch_meta_bytes=229264 combine_payload_bytes=159457280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159686544 ingress_bytes=184623104 link_bytes=184623104 token_bytes=319143824 weight_movement_bytes=25165824 total_nvlink_bytes=344309648 weight_to_token_ratio=0.078854 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404768501e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158851072 dispatch_meta_bytes=229392 combine_payload_bytes=158851072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159080464 ingress_bytes=184016896 link_bytes=184016896 token_bytes=317931536 weight_movement_bytes=25165824 total_nvlink_bytes=343097360 weight_to_token_ratio=0.079155 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400155959e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158859264 dispatch_meta_bytes=229184 combine_payload_bytes=158859264 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159088448 ingress_bytes=234356736 link_bytes=234356736 token_bytes=317947712 weight_movement_bytes=75497472 total_nvlink_bytes=393445184 weight_to_token_ratio=0.237452 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783183977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159326208 dispatch_meta_bytes=228984 combine_payload_bytes=159326208 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159555192 ingress_bytes=184492032 link_bytes=184492032 token_bytes=318881400 weight_movement_bytes=25165824 total_nvlink_bytes=344047224 weight_to_token_ratio=0.078919 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403771195e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159514624 dispatch_meta_bytes=228480 combine_payload_bytes=159514624 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159743104 ingress_bytes=235012096 link_bytes=235012096 token_bytes=319257728 weight_movement_bytes=75497472 total_nvlink_bytes=394755200 weight_to_token_ratio=0.236478 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788170509e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38846 dup_rows=21034 dup_rows_remote=18244 dedup_ratio=0.320953 router_tokens_per_expert=[475, 502, 514, 517, 526, 519, 565, 515, 549, 502, 470, 533, 530, 524, 523, 512, 555, 535, 542, 492, 481, 504, 534, 481, 495, 498, 499, 491, 512, 479, 514, 519, 488, 503, 466, 510, 480, 499, 568, 529, 511, 494, 484, 506, 531, 537, 479, 498, 516, 524, 478, 484, 513, 529, 512, 502, 552, 505, 522, 544, 518, 535, 512, 532] payload_rows_per_dst=[5640, 5556, 5532, 5454, 5500, 5564, 5600, 5656] dup_rows_per_dst=[2626, 2730, 2564, 2560, 2586, 2526, 2652, 2790] combine_rows_per_src=[5640, 5556, 5532, 5454, 5500, 5564, 5600, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44410 payload_rows_remote=38890 dup_rows=21126 dup_rows_remote=18492 dedup_ratio=0.322357 router_tokens_per_expert=[523, 546, 539, 539, 521, 527, 520, 502, 512, 474, 517, 497, 515, 529, 512, 523, 551, 523, 515, 506, 520, 515, 509, 560, 482, 517, 523, 531, 558, 473, 491, 505, 490, 470, 487, 527, 525, 467, 526, 525, 521, 535, 470, 520, 519, 533, 514, 465, 496, 539, 499, 491, 490, 514, 470, 515, 513, 511, 522, 508, 500, 543, 490, 498] payload_rows_per_dst=[5648, 5554, 5666, 5560, 5438, 5520, 5492, 5532] dup_rows_per_dst=[2786, 2604, 2732, 2600, 2596, 2634, 2536, 2638] combine_rows_per_src=[5648, 5554, 5666, 5560, 5438, 5520, 5492, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44396 payload_rows_remote=38830 dup_rows=21140 dup_rows_remote=18398 dedup_ratio=0.322571 router_tokens_per_expert=[475, 529, 513, 521, 524, 511, 486, 507, 522, 503, 508, 472, 493, 523, 541, 505, 512, 491, 544, 486, 503, 494, 552, 479, 535, 528, 503, 521, 518, 521, 505, 535, 517, 480, 562, 502, 532, 528, 518, 515, 489, 520, 510, 490, 528, 498, 509, 567, 534, 483, 505, 516, 543, 510, 530, 506, 477, 517, 452, 493, 552, 533, 510, 482] payload_rows_per_dst=[5536, 5522, 5516, 5654, 5566, 5560, 5546, 5496] dup_rows_per_dst=[2596, 2612, 2606, 2678, 2742, 2662, 2708, 2536] combine_rows_per_src=[5536, 5522, 5516, 5654, 5566, 5560, 5546, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=228360 combine_payload_bytes=159113216 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159341576 ingress_bytes=234610688 link_bytes=234610688 token_bytes=318454792 weight_movement_bytes=75497472 total_nvlink_bytes=393952264 weight_to_token_ratio=0.237074 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785116258e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=229528 combine_payload_bytes=159293440 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159522968 ingress_bytes=209625088 link_bytes=209625088 token_bytes=318816408 weight_movement_bytes=50331648 total_nvlink_bytes=369148056 weight_to_token_ratio=0.157870 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595004711e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159047680 dispatch_meta_bytes=228912 combine_payload_bytes=159047680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159276592 ingress_bytes=184213504 link_bytes=184213504 token_bytes=318324272 weight_movement_bytes=25165824 total_nvlink_bytes=343490096 weight_to_token_ratio=0.079057 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401651918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=38990 dup_rows=20988 dup_rows_remote=18270 dedup_ratio=0.320251 router_tokens_per_expert=[521, 514, 509, 492, 513, 532, 509, 507, 520, 482, 535, 537, 535, 535, 491, 509, 548, 522, 559, 470, 495, 515, 517, 527, 526, 516, 517, 517, 510, 528, 499, 489, 539, 509, 544, 534, 518, 503, 481, 493, 537, 481, 527, 468, 493, 517, 520, 527, 515, 506, 498, 505, 519, 518, 510, 504, 491, 492, 491, 487, 487, 539, 512, 507] payload_rows_per_dst=[5558, 5634, 5632, 5568, 5600, 5556, 5540, 5460] dup_rows_per_dst=[2718, 2654, 2674, 2662, 2720, 2506, 2610, 2444] combine_rows_per_src=[5558, 5634, 5632, 5568, 5600, 5556, 5540, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38922 dup_rows=21036 dup_rows_remote=18424 dedup_ratio=0.320984 router_tokens_per_expert=[487, 508, 504, 490, 490, 522, 521, 518, 539, 482, 513, 484, 521, 540, 496, 520, 531, 483, 481, 520, 481, 526, 501, 518, 499, 544, 516, 509, 494, 492, 538, 517, 494, 495, 516, 487, 537, 523, 509, 538, 549, 501, 473, 514, 501, 532, 549, 521, 493, 496, 525, 533, 531, 514, 485, 509, 556, 514, 497, 512, 520, 535, 508, 516] payload_rows_per_dst=[5502, 5578, 5508, 5548, 5626, 5576, 5516, 5646] dup_rows_per_dst=[2578, 2612, 2574, 2670, 2572, 2704, 2656, 2670] combine_rows_per_src=[5502, 5578, 5508, 5548, 5626, 5576, 5516, 5646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=38928 dup_rows=21070 dup_rows_remote=18458 dedup_ratio=0.321503 router_tokens_per_expert=[534, 490, 502, 522, 478, 540, 517, 507, 540, 519, 506, 494, 518, 506, 526, 536, 484, 533, 501, 532, 508, 529, 510, 497, 519, 511, 527, 505, 490, 489, 501, 533, 504, 504, 513, 496, 557, 480, 527, 467, 467, 490, 532, 523, 513, 513, 538, 506, 522, 527, 520, 517, 503, 511, 545, 514, 507, 497, 498, 513, 491, 497, 554, 518] payload_rows_per_dst=[5562, 5646, 5524, 5538, 5490, 5534, 5630, 5542] dup_rows_per_dst=[2618, 2644, 2664, 2612, 2606, 2630, 2688, 2608] combine_rows_per_src=[5562, 5646, 5524, 5538, 5490, 5534, 5630, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44306 payload_rows_remote=38752 dup_rows=21230 dup_rows_remote=18528 dedup_ratio=0.323944 router_tokens_per_expert=[512, 462, 567, 526, 493, 526, 534, 500, 510, 548, 540, 501, 518, 506, 520, 515, 520, 504, 525, 518, 531, 478, 518, 533, 527, 497, 514, 478, 515, 473, 484, 545, 522, 490, 518, 486, 495, 525, 511, 528, 524, 526, 506, 486, 503, 499, 530, 495, 503, 500, 540, 523, 531, 489, 517, 525, 507, 480, 502, 499, 529, 518, 521, 502] payload_rows_per_dst=[5590, 5586, 5548, 5510, 5446, 5584, 5554, 5488] dup_rows_per_dst=[2650, 2730, 2706, 2556, 2704, 2554, 2702, 2628] combine_rows_per_src=[5590, 5586, 5548, 5510, 5446, 5584, 5554, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=39008 dup_rows=21070 dup_rows_remote=18392 dedup_ratio=0.321503 router_tokens_per_expert=[506, 511, 504, 536, 516, 512, 493, 481, 475, 507, 487, 546, 527, 498, 510, 502, 517, 487, 499, 508, 539, 522, 483, 513, 507, 521, 506, 494, 511, 513, 528, 543, 560, 503, 508, 524, 511, 499, 536, 483, 505, 518, 516, 541, 528, 519, 523, 502, 512, 519, 501, 507, 471, 502, 516, 508, 526, 525, 515, 512, 525, 493, 550, 508] payload_rows_per_dst=[5554, 5528, 5458, 5580, 5608, 5604, 5510, 5624] dup_rows_per_dst=[2564, 2576, 2678, 2666, 2640, 2700, 2562, 2684] combine_rows_per_src=[5554, 5528, 5458, 5580, 5608, 5604, 5510, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159703040 dispatch_meta_bytes=229040 combine_payload_bytes=159703040 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159932080 ingress_bytes=184868864 link_bytes=184868864 token_bytes=319635120 weight_movement_bytes=25165824 total_nvlink_bytes=344800944 weight_to_token_ratio=0.078733 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406638451e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=229384 combine_payload_bytes=159424512 prefetch_bytes=0 reduce_bytes=0 egress_bytes=159653896 ingress_bytes=159424512 link_bytes=159653896 token_bytes=319078408 weight_movement_bytes=0 total_nvlink_bytes=319078408 weight_to_token_ratio=0.000000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.214781679e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159449088 dispatch_meta_bytes=229544 combine_payload_bytes=159449088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159678632 ingress_bytes=184614912 link_bytes=184614912 token_bytes=319127720 weight_movement_bytes=25165824 total_nvlink_bytes=344293544 weight_to_token_ratio=0.078858 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404706170e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=38970 dup_rows=20916 dup_rows_remote=18190 dedup_ratio=0.319153 router_tokens_per_expert=[495, 498, 531, 512, 513, 530, 517, 535, 506, 505, 529, 455, 520, 532, 507, 481, 482, 529, 524, 503, 484, 539, 477, 479, 528, 491, 540, 545, 509, 532, 513, 516, 503, 534, 505, 508, 458, 466, 522, 524, 535, 498, 529, 514, 473, 543, 511, 488, 527, 518, 497, 527, 532, 525, 524, 477, 528, 532, 546, 513, 522, 498, 532, 502] payload_rows_per_dst=[5344, 5400, 5402, 5802, 5750, 5688, 5584, 5650] dup_rows_per_dst=[2286, 2362, 2244, 2824, 2922, 2882, 2670, 2726] combine_rows_per_src=[5344, 5400, 5402, 5802, 5750, 5688, 5584, 5650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158728192 dispatch_meta_bytes=229120 combine_payload_bytes=158728192 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158957312 ingress_bytes=209059840 link_bytes=209059840 token_bytes=317685504 weight_movement_bytes=50331648 total_nvlink_bytes=368017152 weight_to_token_ratio=0.158432 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590703827e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=39088 dup_rows=20982 dup_rows_remote=18406 dedup_ratio=0.320160 router_tokens_per_expert=[523, 529, 495, 534, 519, 529, 552, 522, 482, 504, 510, 523, 525, 549, 522, 512, 537, 541, 496, 534, 505, 513, 512, 533, 541, 492, 465, 492, 521, 507, 491, 497, 485, 503, 521, 505, 501, 512, 510, 484, 526, 514, 472, 506, 507, 503, 529, 521, 492, 475, 527, 515, 511, 538, 527, 471, 542, 514, 520, 554, 485, 496, 501, 494] payload_rows_per_dst=[5686, 5598, 5620, 5476, 5466, 5616, 5494, 5598] dup_rows_per_dst=[2720, 2656, 2722, 2536, 2576, 2540, 2618, 2614] combine_rows_per_src=[5686, 5598, 5620, 5476, 5466, 5616, 5494, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44778 payload_rows_remote=39220 dup_rows=20758 dup_rows_remote=18296 dedup_ratio=0.316742 router_tokens_per_expert=[537, 485, 508, 501, 519, 523, 519, 491, 476, 555, 512, 492, 535, 522, 541, 527, 519, 503, 491, 523, 500, 503, 544, 552, 521, 467, 495, 494, 491, 501, 541, 519, 526, 540, 529, 514, 506, 522, 481, 502, 513, 490, 473, 525, 486, 522, 504, 497, 525, 532, 496, 524, 485, 518, 507, 532, 520, 501, 534, 519, 484, 520, 497, 537] payload_rows_per_dst=[5620, 5644, 5602, 5522, 5602, 5558, 5604, 5626] dup_rows_per_dst=[2546, 2676, 2668, 2536, 2638, 2462, 2634, 2598] combine_rows_per_src=[5620, 5644, 5602, 5522, 5602, 5558, 5604, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159776768 dispatch_meta_bytes=229600 combine_payload_bytes=159776768 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160006368 ingress_bytes=235274240 link_bytes=235274240 token_bytes=319783136 weight_movement_bytes=75497472 total_nvlink_bytes=395280608 weight_to_token_ratio=0.236090 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790165122e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159621120 dispatch_meta_bytes=228640 combine_payload_bytes=159621120 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159849760 ingress_bytes=184786944 link_bytes=184786944 token_bytes=319470880 weight_movement_bytes=25165824 total_nvlink_bytes=344636704 weight_to_token_ratio=0.078773 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406015134e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160104448 dispatch_meta_bytes=229976 combine_payload_bytes=160104448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160334424 ingress_bytes=185270272 link_bytes=185270272 token_bytes=320438872 weight_movement_bytes=25165824 total_nvlink_bytes=345604696 weight_to_token_ratio=0.078535 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409692702e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160645120 dispatch_meta_bytes=230064 combine_payload_bytes=160645120 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160875184 ingress_bytes=185810944 link_bytes=185810944 token_bytes=321520304 weight_movement_bytes=25165824 total_nvlink_bytes=346686128 weight_to_token_ratio=0.078271 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.413806591e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44362 payload_rows_remote=38840 dup_rows=21174 dup_rows_remote=18490 dedup_ratio=0.323090 router_tokens_per_expert=[509, 520, 496, 501, 505, 522, 479, 509, 511, 493, 519, 495, 490, 523, 537, 519, 525, 529, 517, 500, 546, 526, 507, 558, 511, 472, 486, 497, 510, 497, 524, 499, 514, 526, 460, 496, 532, 504, 501, 506, 515, 529, 536, 489, 514, 549, 535, 488, 527, 537, 535, 520, 520, 504, 525, 543, 518, 534, 487, 529, 512, 462, 486, 503] payload_rows_per_dst=[5522, 5606, 5598, 5454, 5494, 5586, 5596, 5506] dup_rows_per_dst=[2684, 2584, 2806, 2538, 2584, 2724, 2698, 2556] combine_rows_per_src=[5522, 5606, 5598, 5454, 5494, 5586, 5596, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38940 dup_rows=21000 dup_rows_remote=18256 dedup_ratio=0.320435 router_tokens_per_expert=[513, 523, 515, 531, 494, 496, 513, 545, 512, 517, 498, 527, 510, 585, 535, 486, 508, 500, 502, 519, 509, 519, 523, 505, 513, 511, 511, 545, 515, 522, 507, 485, 540, 535, 508, 517, 516, 518, 486, 520, 512, 499, 467, 513, 517, 502, 502, 536, 511, 512, 520, 481, 509, 479, 502, 539, 515, 485, 514, 497, 514, 490, 516, 502] payload_rows_per_dst=[5576, 5596, 5572, 5582, 5632, 5572, 5488, 5518] dup_rows_per_dst=[2684, 2744, 2598, 2636, 2648, 2524, 2618, 2548] combine_rows_per_src=[5576, 5596, 5572, 5582, 5632, 5572, 5488, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38864 dup_rows=21088 dup_rows_remote=18328 dedup_ratio=0.321777 router_tokens_per_expert=[524, 539, 509, 507, 515, 537, 529, 514, 549, 529, 552, 531, 446, 523, 486, 499, 551, 502, 470, 478, 537, 493, 526, 509, 531, 556, 538, 516, 496, 482, 528, 525, 493, 460, 523, 508, 503, 480, 546, 512, 486, 490, 532, 481, 497, 518, 450, 514, 532, 497, 500, 540, 511, 522, 524, 506, 513, 513, 509, 523, 528, 492, 522, 516] payload_rows_per_dst=[5634, 5574, 5526, 5584, 5522, 5428, 5600, 5580] dup_rows_per_dst=[2714, 2656, 2606, 2760, 2528, 2508, 2664, 2652] combine_rows_per_src=[5634, 5574, 5526, 5584, 5522, 5428, 5600, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38798 dup_rows=21102 dup_rows_remote=18450 dedup_ratio=0.321991 router_tokens_per_expert=[522, 491, 536, 506, 491, 493, 503, 534, 503, 546, 504, 520, 514, 498, 519, 484, 494, 529, 499, 463, 506, 512, 483, 561, 490, 495, 502, 565, 510, 544, 491, 517, 496, 486, 525, 483, 503, 525, 519, 526, 508, 521, 482, 515, 468, 537, 544, 502, 540, 521, 545, 520, 488, 511, 492, 527, 490, 552, 520, 515, 515, 542, 497, 528] payload_rows_per_dst=[5516, 5506, 5516, 5558, 5506, 5544, 5636, 5652] dup_rows_per_dst=[2636, 2670, 2578, 2670, 2620, 2610, 2652, 2666] combine_rows_per_src=[5516, 5506, 5516, 5558, 5506, 5544, 5636, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44592 payload_rows_remote=38956 dup_rows=20944 dup_rows_remote=18262 dedup_ratio=0.319580 router_tokens_per_expert=[540, 508, 484, 507, 526, 516, 510, 549, 523, 514, 517, 482, 501, 480, 525, 520, 513, 527, 494, 500, 559, 513, 504, 549, 509, 522, 529, 508, 525, 491, 535, 491, 475, 472, 519, 485, 522, 552, 505, 482, 484, 535, 501, 543, 493, 518, 538, 543, 463, 518, 515, 497, 495, 503, 501, 514, 477, 533, 523, 517, 549, 498, 506, 521] payload_rows_per_dst=[5644, 5576, 5636, 5548, 5496, 5626, 5474, 5592] dup_rows_per_dst=[2636, 2548, 2682, 2672, 2528, 2684, 2538, 2656] combine_rows_per_src=[5644, 5576, 5636, 5548, 5496, 5626, 5474, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=229320 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159317960 ingress_bytes=209420288 link_bytes=209420288 token_bytes=318406600 weight_movement_bytes=50331648 total_nvlink_bytes=368738248 weight_to_token_ratio=0.158074 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593446420e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=228784 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159727024 ingress_bytes=234995712 link_bytes=234995712 token_bytes=319225264 weight_movement_bytes=75497472 total_nvlink_bytes=394722736 weight_to_token_ratio=0.236502 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788045846e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159186944 dispatch_meta_bytes=228768 combine_payload_bytes=159186944 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159415712 ingress_bytes=184352768 link_bytes=184352768 token_bytes=318602656 weight_movement_bytes=25165824 total_nvlink_bytes=343768480 weight_to_token_ratio=0.078988 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402711557e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44518 payload_rows_remote=38940 dup_rows=21018 dup_rows_remote=18294 dedup_ratio=0.320709 router_tokens_per_expert=[521, 509, 515, 505, 499, 494, 467, 536, 492, 533, 501, 502, 546, 506, 502, 507, 491, 498, 499, 512, 507, 500, 553, 512, 522, 503, 524, 505, 521, 511, 524, 509, 520, 522, 479, 500, 541, 494, 535, 501, 529, 529, 500, 518, 539, 506, 503, 549, 513, 494, 534, 522, 498, 545, 490, 499, 531, 512, 504, 493, 485, 518, 490, 549] payload_rows_per_dst=[5380, 5536, 5596, 5624, 5564, 5646, 5594, 5578] dup_rows_per_dst=[2448, 2694, 2548, 2652, 2620, 2700, 2632, 2724] combine_rows_per_src=[5380, 5536, 5596, 5624, 5564, 5646, 5594, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158916608 dispatch_meta_bytes=228992 combine_payload_bytes=158916608 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159145600 ingress_bytes=209248256 link_bytes=209248256 token_bytes=318062208 weight_movement_bytes=50331648 total_nvlink_bytes=368393856 weight_to_token_ratio=0.158245 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592137455e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38940 dup_rows=21076 dup_rows_remote=18528 dedup_ratio=0.321594 router_tokens_per_expert=[497, 555, 485, 551, 503, 524, 489, 509, 500, 490, 486, 533, 491, 511, 498, 498, 524, 518, 513, 465, 508, 471, 530, 531, 531, 537, 558, 511, 506, 494, 496, 501, 511, 522, 497, 518, 489, 498, 497, 502, 513, 557, 511, 496, 503, 567, 522, 505, 498, 502, 528, 522, 537, 494, 546, 518, 480, 504, 520, 504, 514, 509, 541, 529] payload_rows_per_dst=[5552, 5448, 5544, 5592, 5520, 5670, 5624, 5510] dup_rows_per_dst=[2674, 2566, 2576, 2676, 2548, 2678, 2666, 2692] combine_rows_per_src=[5552, 5448, 5544, 5592, 5520, 5670, 5624, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44588 payload_rows_remote=39138 dup_rows=20948 dup_rows_remote=18314 dedup_ratio=0.319641 router_tokens_per_expert=[530, 478, 544, 498, 538, 506, 516, 520, 542, 535, 555, 505, 509, 495, 496, 520, 495, 528, 533, 488, 500, 484, 536, 531, 515, 519, 516, 521, 512, 512, 514, 517, 537, 472, 476, 512, 514, 482, 500, 482, 510, 477, 525, 504, 505, 502, 504, 515, 527, 511, 509, 530, 525, 505, 515, 552, 468, 521, 502, 529, 493, 530, 530, 496] payload_rows_per_dst=[5600, 5660, 5582, 5552, 5514, 5450, 5656, 5574] dup_rows_per_dst=[2660, 2654, 2608, 2700, 2436, 2634, 2692, 2564] combine_rows_per_src=[5600, 5660, 5582, 5552, 5514, 5450, 5656, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=228872 combine_payload_bytes=159563776 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159792648 ingress_bytes=235061248 link_bytes=235061248 token_bytes=319356424 weight_movement_bytes=75497472 total_nvlink_bytes=394853896 weight_to_token_ratio=0.236405 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788544499e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=228936 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159727176 ingress_bytes=184664064 link_bytes=184664064 token_bytes=319225416 weight_movement_bytes=25165824 total_nvlink_bytes=344391240 weight_to_token_ratio=0.078834 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405080160e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=229872 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159728112 ingress_bytes=184664064 link_bytes=184664064 token_bytes=319226352 weight_movement_bytes=25165824 total_nvlink_bytes=344392176 weight_to_token_ratio=0.078834 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405080160e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160309248 dispatch_meta_bytes=229808 combine_payload_bytes=160309248 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160539056 ingress_bytes=210640896 link_bytes=210640896 token_bytes=320848304 weight_movement_bytes=50331648 total_nvlink_bytes=371179952 weight_to_token_ratio=0.156871 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602733836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38714 dup_rows=21120 dup_rows_remote=18244 dedup_ratio=0.322266 router_tokens_per_expert=[506, 564, 497, 528, 535, 554, 548, 555, 506, 515, 502, 506, 527, 482, 516, 497, 502, 513, 505, 493, 496, 506, 508, 505, 481, 482, 519, 549, 497, 527, 477, 519, 530, 477, 531, 525, 530, 483, 531, 557, 507, 506, 516, 500, 474, 543, 523, 510, 541, 505, 500, 557, 546, 522, 445, 514, 476, 486, 484, 467, 531, 521, 511, 502] payload_rows_per_dst=[5702, 5522, 5646, 5524, 5784, 5406, 5392, 5440] dup_rows_per_dst=[2876, 2580, 2844, 2578, 2932, 2314, 2480, 2516] combine_rows_per_src=[5702, 5522, 5646, 5524, 5784, 5406, 5392, 5440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44296 payload_rows_remote=38822 dup_rows=21240 dup_rows_remote=18644 dedup_ratio=0.324097 router_tokens_per_expert=[544, 521, 518, 500, 511, 521, 539, 484, 514, 532, 474, 502, 486, 497, 517, 513, 500, 507, 504, 473, 502, 507, 592, 524, 531, 527, 556, 493, 514, 465, 497, 547, 467, 502, 498, 506, 492, 512, 488, 504, 563, 484, 496, 533, 496, 505, 524, 517, 526, 512, 525, 539, 508, 522, 494, 512, 546, 542, 491, 513, 516, 477, 511, 535] payload_rows_per_dst=[5598, 5474, 5574, 5566, 5348, 5576, 5600, 5560] dup_rows_per_dst=[2678, 2596, 2644, 2694, 2590, 2660, 2676, 2702] combine_rows_per_src=[5598, 5474, 5574, 5566, 5348, 5576, 5600, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=39048 dup_rows=20988 dup_rows_remote=18372 dedup_ratio=0.320251 router_tokens_per_expert=[482, 499, 485, 537, 515, 513, 505, 529, 534, 524, 531, 461, 562, 512, 523, 509, 497, 517, 498, 479, 489, 550, 519, 507, 479, 502, 511, 505, 512, 494, 537, 518, 538, 515, 501, 491, 533, 527, 494, 540, 509, 536, 529, 539, 492, 538, 532, 527, 508, 494, 512, 454, 525, 488, 496, 521, 533, 493, 483, 519, 493, 540, 546, 487] payload_rows_per_dst=[5548, 5588, 5516, 5500, 5642, 5648, 5498, 5608] dup_rows_per_dst=[2582, 2724, 2596, 2616, 2636, 2756, 2498, 2580] combine_rows_per_src=[5548, 5588, 5516, 5500, 5642, 5648, 5498, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38852 dup_rows=21088 dup_rows_remote=18352 dedup_ratio=0.321777 router_tokens_per_expert=[506, 528, 523, 498, 467, 506, 521, 519, 477, 512, 501, 489, 547, 511, 512, 513, 480, 467, 515, 515, 542, 567, 514, 502, 513, 493, 500, 481, 507, 509, 514, 518, 519, 544, 512, 494, 515, 503, 513, 500, 545, 497, 523, 500, 544, 504, 501, 550, 497, 488, 513, 542, 551, 532, 506, 537, 495, 532, 459, 543, 514, 509, 506, 513] payload_rows_per_dst=[5528, 5502, 5592, 5468, 5612, 5650, 5596, 5500] dup_rows_per_dst=[2608, 2622, 2612, 2602, 2588, 2678, 2736, 2642] combine_rows_per_src=[5528, 5502, 5592, 5468, 5612, 5650, 5596, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=38986 dup_rows=20976 dup_rows_remote=18382 dedup_ratio=0.320068 router_tokens_per_expert=[509, 480, 543, 517, 511, 497, 490, 496, 539, 512, 517, 527, 509, 493, 541, 521, 513, 540, 503, 533, 501, 476, 507, 511, 531, 491, 537, 453, 489, 527, 484, 505, 516, 510, 528, 522, 549, 518, 512, 483, 498, 525, 484, 513, 540, 541, 495, 526, 503, 515, 531, 520, 511, 536, 519, 485, 508, 507, 487, 552, 500, 507, 496, 528] payload_rows_per_dst=[5462, 5632, 5574, 5472, 5608, 5652, 5580, 5580] dup_rows_per_dst=[2624, 2686, 2594, 2562, 2668, 2592, 2660, 2590] combine_rows_per_src=[5462, 5632, 5574, 5472, 5608, 5652, 5580, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38774 dup_rows=21146 dup_rows_remote=18422 dedup_ratio=0.322662 router_tokens_per_expert=[505, 477, 501, 530, 492, 518, 514, 514, 501, 510, 528, 506, 490, 509, 516, 523, 517, 483, 537, 482, 504, 528, 517, 513, 531, 511, 519, 491, 520, 497, 460, 509, 490, 533, 538, 504, 507, 539, 492, 545, 530, 484, 489, 519, 529, 522, 509, 541, 531, 492, 534, 475, 528, 528, 522, 510, 514, 528, 515, 521, 502, 505, 511, 528] payload_rows_per_dst=[5438, 5550, 5522, 5510, 5434, 5588, 5732, 5616] dup_rows_per_dst=[2664, 2616, 2548, 2566, 2342, 2658, 3028, 2724] combine_rows_per_src=[5438, 5550, 5522, 5510, 5434, 5588, 5732, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159014912 dispatch_meta_bytes=229864 combine_payload_bytes=159014912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159244776 ingress_bytes=184180736 link_bytes=184180736 token_bytes=318259688 weight_movement_bytes=25165824 total_nvlink_bytes=343425512 weight_to_token_ratio=0.079073 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401402592e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158572544 dispatch_meta_bytes=227832 combine_payload_bytes=158572544 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=158800376 ingress_bytes=284401664 link_bytes=284401664 token_bytes=317372920 weight_movement_bytes=125829120 total_nvlink_bytes=443202040 weight_to_token_ratio=0.396471 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.163968055e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159940608 dispatch_meta_bytes=229680 combine_payload_bytes=159940608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160170288 ingress_bytes=185106432 link_bytes=185106432 token_bytes=320110896 weight_movement_bytes=25165824 total_nvlink_bytes=345276720 weight_to_token_ratio=0.078616 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408446069e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159137792 dispatch_meta_bytes=228816 combine_payload_bytes=159137792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159366608 ingress_bytes=184303616 link_bytes=184303616 token_bytes=318504400 weight_movement_bytes=25165824 total_nvlink_bytes=343670224 weight_to_token_ratio=0.079012 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402337567e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=39032 dup_rows=20982 dup_rows_remote=18166 dedup_ratio=0.320160 router_tokens_per_expert=[515, 519, 504, 504, 500, 525, 523, 493, 492, 476, 502, 493, 528, 507, 520, 545, 505, 527, 527, 553, 539, 488, 493, 517, 527, 510, 501, 497, 495, 503, 538, 532, 492, 513, 533, 519, 562, 520, 534, 496, 537, 499, 476, 533, 496, 520, 489, 537, 512, 469, 491, 474, 486, 501, 518, 520, 549, 511, 521, 516, 532, 503, 504, 507] payload_rows_per_dst=[5638, 5552, 5688, 5514, 5522, 5560, 5476, 5604] dup_rows_per_dst=[2528, 2574, 2610, 2692, 2816, 2614, 2466, 2682] combine_rows_per_src=[5638, 5552, 5688, 5514, 5522, 5560, 5476, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44762 payload_rows_remote=39152 dup_rows=20774 dup_rows_remote=18054 dedup_ratio=0.316986 router_tokens_per_expert=[498, 528, 506, 524, 523, 506, 491, 500, 512, 549, 488, 530, 512, 524, 468, 513, 514, 520, 514, 484, 506, 489, 502, 538, 531, 547, 521, 490, 497, 512, 500, 527, 491, 533, 494, 502, 557, 513, 503, 507, 513, 537, 555, 512, 518, 508, 517, 505, 449, 492, 512, 507, 518, 512, 508, 469, 527, 528, 487, 539, 497, 525, 516, 553] payload_rows_per_dst=[5572, 5596, 5626, 5604, 5602, 5610, 5538, 5614] dup_rows_per_dst=[2580, 2596, 2508, 2646, 2598, 2720, 2396, 2730] combine_rows_per_src=[5572, 5596, 5626, 5604, 5602, 5610, 5538, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158818304 dispatch_meta_bytes=228784 combine_payload_bytes=158818304 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159047088 ingress_bytes=209149952 link_bytes=209149952 token_bytes=317865392 weight_movement_bytes=50331648 total_nvlink_bytes=368197040 weight_to_token_ratio=0.158343 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591389475e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159686656 dispatch_meta_bytes=229472 combine_payload_bytes=159686656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159916128 ingress_bytes=184852480 link_bytes=184852480 token_bytes=319602784 weight_movement_bytes=25165824 total_nvlink_bytes=344768608 weight_to_token_ratio=0.078741 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406513788e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159875072 dispatch_meta_bytes=228792 combine_payload_bytes=159875072 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160103864 ingress_bytes=210206720 link_bytes=210206720 token_bytes=319978936 weight_movement_bytes=50331648 total_nvlink_bytes=370310584 weight_to_token_ratio=0.157297 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599430259e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160366592 dispatch_meta_bytes=228824 combine_payload_bytes=160366592 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160595416 ingress_bytes=210698240 link_bytes=210698240 token_bytes=320962008 weight_movement_bytes=50331648 total_nvlink_bytes=371293656 weight_to_token_ratio=0.156815 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.603170158e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44584 payload_rows_remote=38920 dup_rows=20952 dup_rows_remote=18282 dedup_ratio=0.319702 router_tokens_per_expert=[476, 540, 478, 521, 456, 490, 495, 512, 541, 542, 525, 516, 533, 521, 483, 506, 536, 510, 516, 509, 529, 521, 524, 485, 519, 537, 497, 551, 469, 508, 539, 509, 548, 540, 492, 527, 541, 528, 522, 511, 524, 517, 540, 480, 508, 473, 487, 508, 514, 508, 519, 510, 489, 519, 496, 495, 511, 482, 524, 514, 489, 514, 491, 553] payload_rows_per_dst=[5442, 5664, 5562, 5632, 5700, 5516, 5516, 5552] dup_rows_per_dst=[2494, 2670, 2698, 2626, 2718, 2558, 2584, 2604] combine_rows_per_src=[5442, 5664, 5562, 5632, 5700, 5516, 5516, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44616 payload_rows_remote=38850 dup_rows=20920 dup_rows_remote=17926 dedup_ratio=0.319214 router_tokens_per_expert=[529, 515, 480, 557, 543, 522, 489, 535, 517, 519, 486, 501, 502, 552, 488, 507, 518, 513, 544, 503, 491, 467, 489, 516, 478, 519, 526, 547, 548, 477, 517, 476, 526, 488, 507, 499, 512, 538, 498, 523, 537, 555, 487, 562, 503, 488, 500, 524, 482, 534, 516, 489, 499, 510, 498, 531, 512, 533, 492, 511, 506, 498, 514, 525] payload_rows_per_dst=[5766, 5574, 5600, 5570, 5584, 5524, 5580, 5418] dup_rows_per_dst=[2994, 2594, 2512, 2560, 2590, 2594, 2538, 2538] combine_rows_per_src=[5766, 5574, 5600, 5570, 5584, 5524, 5580, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=38978 dup_rows=21014 dup_rows_remote=18322 dedup_ratio=0.320648 router_tokens_per_expert=[516, 508, 509, 528, 514, 495, 478, 484, 524, 522, 501, 493, 516, 524, 518, 500, 493, 540, 529, 501, 489, 521, 554, 539, 505, 535, 526, 494, 485, 501, 565, 507, 501, 492, 531, 542, 484, 511, 518, 504, 487, 521, 502, 508, 494, 515, 511, 510, 516, 506, 524, 501, 531, 549, 518, 488, 544, 511, 497, 475, 516, 507, 532, 508] payload_rows_per_dst=[5494, 5630, 5632, 5544, 5566, 5540, 5588, 5528] dup_rows_per_dst=[2570, 2566, 2700, 2692, 2600, 2556, 2678, 2652] combine_rows_per_src=[5494, 5630, 5632, 5544, 5566, 5540, 5588, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=39018 dup_rows=21026 dup_rows_remote=18482 dedup_ratio=0.320831 router_tokens_per_expert=[509, 567, 490, 506, 491, 525, 514, 526, 531, 490, 509, 519, 533, 519, 550, 511, 527, 483, 484, 502, 495, 525, 517, 519, 522, 542, 473, 518, 495, 541, 504, 486, 519, 493, 502, 510, 480, 517, 523, 531, 508, 529, 515, 517, 515, 517, 509, 525, 468, 517, 497, 488, 524, 484, 534, 506, 541, 521, 523, 493, 548, 519, 491, 481] payload_rows_per_dst=[5616, 5650, 5554, 5564, 5512, 5556, 5492, 5566] dup_rows_per_dst=[2640, 2674, 2550, 2598, 2638, 2714, 2544, 2668] combine_rows_per_src=[5616, 5650, 5554, 5564, 5512, 5556, 5492, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44368 payload_rows_remote=38674 dup_rows=21168 dup_rows_remote=18250 dedup_ratio=0.322998 router_tokens_per_expert=[509, 567, 479, 506, 519, 554, 532, 516, 499, 500, 519, 541, 539, 472, 497, 503, 512, 526, 530, 515, 474, 522, 539, 494, 516, 545, 523, 492, 492, 500, 483, 517, 482, 501, 524, 524, 524, 518, 508, 488, 517, 486, 507, 558, 497, 499, 530, 524, 511, 522, 555, 500, 524, 488, 497, 521, 508, 531, 508, 483, 506, 502, 491, 502] payload_rows_per_dst=[5470, 5574, 5494, 5416, 5620, 5514, 5586, 5694] dup_rows_per_dst=[2680, 2592, 2544, 2342, 2696, 2598, 2798, 2918] combine_rows_per_src=[5470, 5574, 5494, 5416, 5620, 5514, 5586, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=228808 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159645128 ingress_bytes=184582144 link_bytes=184582144 token_bytes=319061448 weight_movement_bytes=25165824 total_nvlink_bytes=344227272 weight_to_token_ratio=0.078875 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404456843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159129600 dispatch_meta_bytes=227104 combine_payload_bytes=159129600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159356704 ingress_bytes=184295424 link_bytes=184295424 token_bytes=318486304 weight_movement_bytes=25165824 total_nvlink_bytes=343652128 weight_to_token_ratio=0.079017 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402275235e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38924 dup_rows=21124 dup_rows_remote=18486 dedup_ratio=0.322327 router_tokens_per_expert=[510, 525, 497, 511, 516, 530, 510, 511, 494, 528, 525, 519, 482, 548, 515, 504, 500, 510, 524, 484, 503, 530, 540, 472, 510, 503, 504, 561, 490, 513, 540, 534, 536, 501, 538, 513, 473, 524, 553, 494, 478, 518, 518, 515, 527, 493, 511, 525, 498, 474, 498, 533, 522, 504, 502, 511, 526, 486, 510, 499, 502, 532, 513, 498] payload_rows_per_dst=[5536, 5532, 5488, 5614, 5622, 5586, 5528, 5506] dup_rows_per_dst=[2684, 2698, 2638, 2696, 2642, 2584, 2556, 2626] combine_rows_per_src=[5536, 5532, 5488, 5614, 5622, 5586, 5528, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=230000 combine_payload_bytes=159817728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160047728 ingress_bytes=184983552 link_bytes=184983552 token_bytes=319865456 weight_movement_bytes=25165824 total_nvlink_bytes=345031280 weight_to_token_ratio=0.078676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407511094e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159653888 dispatch_meta_bytes=229200 combine_payload_bytes=159653888 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159883088 ingress_bytes=235151360 link_bytes=235151360 token_bytes=319536976 weight_movement_bytes=75497472 total_nvlink_bytes=395034448 weight_to_token_ratio=0.236271 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789230147e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38992 dup_rows=21046 dup_rows_remote=18498 dedup_ratio=0.321136 router_tokens_per_expert=[517, 507, 528, 537, 542, 525, 517, 505, 481, 512, 528, 506, 523, 546, 537, 472, 520, 539, 514, 499, 490, 505, 499, 529, 504, 507, 503, 524, 530, 543, 546, 512, 516, 489, 491, 508, 498, 504, 527, 490, 515, 465, 507, 526, 532, 511, 512, 475, 522, 501, 484, 486, 505, 521, 526, 541, 527, 485, 529, 545, 502, 471, 497, 513] payload_rows_per_dst=[5592, 5528, 5572, 5624, 5498, 5574, 5536, 5566] dup_rows_per_dst=[2764, 2682, 2618, 2714, 2548, 2512, 2636, 2572] combine_rows_per_src=[5592, 5528, 5572, 5624, 5498, 5574, 5536, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44482 payload_rows_remote=38914 dup_rows=21054 dup_rows_remote=18370 dedup_ratio=0.321259 router_tokens_per_expert=[513, 496, 531, 500, 496, 530, 527, 514, 555, 498, 497, 511, 512, 507, 538, 492, 539, 491, 535, 490, 499, 532, 521, 531, 501, 486, 500, 539, 529, 528, 509, 513, 520, 514, 506, 489, 531, 482, 520, 554, 511, 532, 509, 537, 516, 487, 518, 516, 492, 522, 515, 511, 480, 532, 543, 504, 543, 487, 486, 478, 505, 497, 509, 462] payload_rows_per_dst=[5550, 5570, 5552, 5642, 5582, 5568, 5556, 5462] dup_rows_per_dst=[2664, 2650, 2724, 2568, 2650, 2684, 2642, 2472] combine_rows_per_src=[5550, 5570, 5552, 5642, 5582, 5568, 5556, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158408704 dispatch_meta_bytes=227696 combine_payload_bytes=158408704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158636400 ingress_bytes=183574528 link_bytes=183574528 token_bytes=317045104 weight_movement_bytes=25165824 total_nvlink_bytes=342210928 weight_to_token_ratio=0.079376 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396790049e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=229640 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159662344 ingress_bytes=209764352 link_bytes=209764352 token_bytes=319095048 weight_movement_bytes=50331648 total_nvlink_bytes=369426696 weight_to_token_ratio=0.157732 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596064349e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159711232 dispatch_meta_bytes=229960 combine_payload_bytes=159711232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159941192 ingress_bytes=184877056 link_bytes=184877056 token_bytes=319652424 weight_movement_bytes=25165824 total_nvlink_bytes=344818248 weight_to_token_ratio=0.078729 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406700783e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159391744 dispatch_meta_bytes=229136 combine_payload_bytes=159391744 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159620880 ingress_bytes=209723392 link_bytes=209723392 token_bytes=319012624 weight_movement_bytes=50331648 total_nvlink_bytes=369344272 weight_to_token_ratio=0.157773 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595752691e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44404 payload_rows_remote=38832 dup_rows=21132 dup_rows_remote=18362 dedup_ratio=0.322449 router_tokens_per_expert=[512, 529, 562, 483, 498, 507, 551, 529, 501, 510, 503, 511, 541, 511, 469, 532, 497, 525, 532, 503, 538, 489, 466, 533, 532, 524, 516, 489, 526, 537, 529, 538, 519, 519, 497, 497, 511, 492, 517, 475, 503, 496, 503, 536, 494, 541, 538, 517, 468, 497, 544, 489, 505, 479, 506, 536, 513, 530, 500, 514, 516, 501, 516, 476] payload_rows_per_dst=[5572, 5550, 5502, 5624, 5526, 5588, 5502, 5540] dup_rows_per_dst=[2770, 2606, 2664, 2758, 2528, 2668, 2546, 2592] combine_rows_per_src=[5572, 5550, 5502, 5624, 5526, 5588, 5502, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=39062 dup_rows=20982 dup_rows_remote=18528 dedup_ratio=0.320160 router_tokens_per_expert=[487, 488, 511, 515, 551, 528, 516, 486, 493, 464, 509, 469, 525, 499, 487, 527, 510, 521, 522, 508, 489, 534, 536, 529, 513, 479, 531, 519, 524, 500, 511, 526, 544, 514, 553, 522, 521, 509, 551, 481, 525, 477, 483, 551, 527, 508, 504, 497, 479, 486, 500, 552, 496, 516, 527, 566, 537, 538, 442, 492, 523, 482, 528, 530] payload_rows_per_dst=[5554, 5492, 5584, 5562, 5608, 5600, 5618, 5536] dup_rows_per_dst=[2610, 2454, 2714, 2644, 2782, 2544, 2626, 2608] combine_rows_per_src=[5554, 5492, 5584, 5562, 5608, 5600, 5618, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44332 payload_rows_remote=38802 dup_rows=21204 dup_rows_remote=18600 dedup_ratio=0.323547 router_tokens_per_expert=[507, 488, 494, 523, 480, 520, 504, 507, 502, 539, 518, 525, 517, 491, 499, 539, 501, 509, 489, 508, 533, 552, 508, 498, 524, 555, 548, 495, 535, 509, 473, 510, 479, 506, 524, 534, 522, 539, 503, 527, 506, 537, 491, 517, 530, 515, 498, 519, 521, 495, 555, 497, 490, 493, 476, 540, 506, 513, 499, 463, 483, 517, 543, 530] payload_rows_per_dst=[5422, 5568, 5548, 5602, 5620, 5532, 5530, 5510] dup_rows_per_dst=[2624, 2692, 2648, 2696, 2648, 2694, 2604, 2598] combine_rows_per_src=[5422, 5568, 5548, 5602, 5620, 5532, 5530, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44396 payload_rows_remote=38748 dup_rows=21140 dup_rows_remote=18476 dedup_ratio=0.322571 router_tokens_per_expert=[472, 492, 493, 506, 516, 500, 479, 485, 534, 484, 515, 511, 511, 527, 483, 512, 512, 513, 478, 512, 525, 512, 541, 490, 495, 533, 527, 515, 544, 521, 510, 511, 506, 498, 533, 514, 510, 558, 487, 507, 506, 523, 524, 524, 513, 515, 510, 509, 531, 564, 514, 510, 506, 510, 510, 486, 486, 513, 546, 532, 500, 547, 507, 510] payload_rows_per_dst=[5406, 5522, 5574, 5648, 5588, 5588, 5514, 5556] dup_rows_per_dst=[2480, 2632, 2592, 2664, 2638, 2660, 2748, 2726] combine_rows_per_src=[5406, 5522, 5574, 5648, 5588, 5588, 5514, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44328 payload_rows_remote=38556 dup_rows=21208 dup_rows_remote=18192 dedup_ratio=0.323608 router_tokens_per_expert=[495, 509, 520, 515, 491, 489, 526, 543, 524, 474, 534, 518, 487, 504, 525, 503, 514, 535, 509, 519, 494, 554, 507, 525, 516, 483, 480, 465, 522, 536, 527, 547, 526, 522, 470, 534, 465, 491, 511, 525, 546, 505, 515, 508, 538, 503, 499, 499, 523, 532, 533, 516, 495, 503, 505, 558, 506, 516, 505, 474, 516, 501, 488, 550] payload_rows_per_dst=[5512, 5550, 5494, 5272, 5458, 5564, 5706, 5772] dup_rows_per_dst=[2658, 2588, 2560, 2270, 2636, 2754, 2726, 3016] combine_rows_per_src=[5512, 5550, 5494, 5272, 5458, 5564, 5706, 5772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159055872 dispatch_meta_bytes=228776 combine_payload_bytes=159055872 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159284648 ingress_bytes=234553344 link_bytes=234553344 token_bytes=318340520 weight_movement_bytes=75497472 total_nvlink_bytes=393837992 weight_to_token_ratio=0.237159 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784679936e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159997952 dispatch_meta_bytes=230360 combine_payload_bytes=159997952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160228312 ingress_bytes=185163776 link_bytes=185163776 token_bytes=320226264 weight_movement_bytes=25165824 total_nvlink_bytes=345392088 weight_to_token_ratio=0.078588 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408882390e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158932992 dispatch_meta_bytes=229608 combine_payload_bytes=158932992 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159162600 ingress_bytes=184098816 link_bytes=184098816 token_bytes=318095592 weight_movement_bytes=25165824 total_nvlink_bytes=343261416 weight_to_token_ratio=0.079114 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400779275e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44656 payload_rows_remote=39006 dup_rows=20880 dup_rows_remote=18232 dedup_ratio=0.318604 router_tokens_per_expert=[505, 513, 480, 532, 531, 508, 493, 522, 520, 516, 539, 521, 513, 509, 536, 526, 483, 545, 506, 516, 523, 546, 524, 506, 490, 518, 526, 500, 515, 512, 499, 524, 517, 501, 506, 509, 520, 498, 505, 505, 497, 527, 521, 517, 506, 497, 482, 537, 497, 492, 518, 551, 522, 476, 538, 502, 483, 471, 503, 498, 486, 536, 522, 531] payload_rows_per_dst=[5570, 5622, 5650, 5566, 5606, 5518, 5602, 5522] dup_rows_per_dst=[2598, 2738, 2648, 2602, 2516, 2650, 2590, 2538] combine_rows_per_src=[5570, 5622, 5650, 5566, 5606, 5518, 5602, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158711808 dispatch_meta_bytes=228896 combine_payload_bytes=158711808 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158940704 ingress_bytes=209043456 link_bytes=209043456 token_bytes=317652512 weight_movement_bytes=50331648 total_nvlink_bytes=367984160 weight_to_token_ratio=0.158449 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590579164e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38930 dup_rows=21050 dup_rows_remote=18388 dedup_ratio=0.321198 router_tokens_per_expert=[508, 483, 520, 516, 521, 470, 503, 533, 521, 485, 514, 548, 496, 511, 522, 554, 505, 532, 544, 529, 510, 528, 568, 490, 500, 499, 492, 490, 531, 487, 543, 529, 496, 525, 534, 497, 505, 521, 539, 492, 532, 507, 501, 543, 522, 535, 481, 506, 500, 503, 498, 502, 525, 493, 480, 498, 508, 508, 524, 499, 515, 512, 492, 493] payload_rows_per_dst=[5554, 5628, 5642, 5530, 5556, 5586, 5466, 5524] dup_rows_per_dst=[2554, 2674, 2770, 2612, 2662, 2668, 2532, 2578] combine_rows_per_src=[5554, 5628, 5642, 5530, 5556, 5586, 5466, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44728 payload_rows_remote=39166 dup_rows=20808 dup_rows_remote=18306 dedup_ratio=0.317505 router_tokens_per_expert=[525, 478, 540, 527, 532, 493, 521, 488, 514, 489, 500, 537, 491, 504, 527, 548, 525, 527, 518, 525, 507, 526, 498, 476, 513, 510, 499, 541, 530, 519, 508, 497, 482, 549, 538, 507, 533, 489, 489, 510, 484, 534, 495, 512, 509, 507, 477, 514, 467, 527, 501, 503, 531, 518, 548, 546, 489, 551, 485, 505, 510, 523, 498, 504] payload_rows_per_dst=[5584, 5620, 5644, 5584, 5516, 5562, 5684, 5534] dup_rows_per_dst=[2624, 2600, 2560, 2650, 2678, 2502, 2598, 2596] combine_rows_per_src=[5584, 5620, 5644, 5584, 5516, 5562, 5684, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=157925376 dispatch_meta_bytes=226992 combine_payload_bytes=157925376 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158152368 ingress_bytes=183091200 link_bytes=183091200 token_bytes=316077744 weight_movement_bytes=25165824 total_nvlink_bytes=341243568 weight_to_token_ratio=0.079619 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.393112482e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159768576 dispatch_meta_bytes=228952 combine_payload_bytes=159768576 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159997528 ingress_bytes=235266048 link_bytes=235266048 token_bytes=319766104 weight_movement_bytes=75497472 total_nvlink_bytes=395263576 weight_to_token_ratio=0.236102 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790102790e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159457280 dispatch_meta_bytes=229272 combine_payload_bytes=159457280 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159686552 ingress_bytes=234954752 link_bytes=234954752 token_bytes=319143832 weight_movement_bytes=75497472 total_nvlink_bytes=394641304 weight_to_token_ratio=0.236563 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787734188e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160423936 dispatch_meta_bytes=229888 combine_payload_bytes=160423936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160653824 ingress_bytes=185589760 link_bytes=185589760 token_bytes=321077760 weight_movement_bytes=25165824 total_nvlink_bytes=346243584 weight_to_token_ratio=0.078379 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412123636e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44686 payload_rows_remote=39056 dup_rows=20850 dup_rows_remote=18244 dedup_ratio=0.318146 router_tokens_per_expert=[521, 535, 471, 526, 528, 514, 518, 505, 525, 522, 519, 517, 495, 515, 497, 500, 516, 480, 491, 508, 468, 509, 517, 477, 529, 508, 518, 501, 511, 509, 476, 531, 491, 522, 567, 509, 532, 543, 503, 504, 509, 499, 499, 503, 500, 553, 533, 505, 521, 509, 495, 501, 536, 518, 485, 515, 512, 519, 522, 507, 502, 565, 508, 524] payload_rows_per_dst=[5630, 5668, 5484, 5580, 5490, 5620, 5556, 5658] dup_rows_per_dst=[2606, 2830, 2482, 2586, 2500, 2582, 2604, 2660] combine_rows_per_src=[5630, 5668, 5484, 5580, 5490, 5620, 5556, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44578 payload_rows_remote=39050 dup_rows=20958 dup_rows_remote=18448 dedup_ratio=0.319794 router_tokens_per_expert=[519, 519, 533, 517, 537, 498, 478, 532, 516, 521, 500, 461, 524, 502, 527, 468, 532, 512, 465, 561, 488, 503, 520, 520, 538, 553, 522, 537, 506, 546, 491, 568, 559, 484, 509, 526, 491, 547, 478, 510, 467, 496, 517, 513, 487, 533, 512, 523, 485, 501, 531, 487, 523, 525, 517, 528, 506, 509, 502, 489, 542, 494, 492, 471] payload_rows_per_dst=[5616, 5528, 5576, 5742, 5570, 5590, 5460, 5496] dup_rows_per_dst=[2650, 2510, 2626, 2780, 2638, 2506, 2734, 2514] combine_rows_per_src=[5616, 5528, 5576, 5742, 5570, 5590, 5460, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44458 payload_rows_remote=39058 dup_rows=21078 dup_rows_remote=18442 dedup_ratio=0.321625 router_tokens_per_expert=[528, 540, 516, 494, 509, 546, 523, 506, 521, 501, 492, 497, 512, 540, 487, 495, 518, 525, 533, 495, 492, 532, 514, 515, 511, 521, 519, 496, 487, 489, 555, 532, 479, 504, 503, 527, 497, 505, 513, 523, 531, 548, 498, 506, 542, 506, 515, 519, 515, 512, 489, 508, 517, 492, 489, 496, 497, 510, 488, 545, 497, 523, 491, 542] payload_rows_per_dst=[5612, 5516, 5616, 5538, 5570, 5608, 5400, 5598] dup_rows_per_dst=[2712, 2574, 2632, 2682, 2532, 2722, 2636, 2588] combine_rows_per_src=[5612, 5516, 5616, 5538, 5570, 5608, 5400, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44674 payload_rows_remote=39064 dup_rows=20862 dup_rows_remote=18134 dedup_ratio=0.318329 router_tokens_per_expert=[535, 477, 532, 515, 503, 517, 501, 541, 507, 503, 506, 511, 525, 486, 471, 504, 526, 486, 551, 506, 510, 503, 536, 514, 544, 519, 534, 514, 489, 537, 536, 496, 504, 474, 515, 537, 500, 558, 506, 533, 499, 514, 503, 497, 525, 509, 497, 518, 533, 505, 520, 520, 537, 502, 496, 486, 503, 494, 519, 519, 542, 466, 498, 504] payload_rows_per_dst=[5594, 5532, 5656, 5610, 5568, 5564, 5610, 5540] dup_rows_per_dst=[2648, 2494, 2608, 2728, 2686, 2560, 2588, 2550] combine_rows_per_src=[5594, 5532, 5656, 5610, 5568, 5564, 5610, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44352 payload_rows_remote=38822 dup_rows=21184 dup_rows_remote=18450 dedup_ratio=0.323242 router_tokens_per_expert=[498, 513, 518, 561, 533, 579, 534, 515, 520, 506, 498, 487, 562, 509, 521, 488, 520, 512, 466, 530, 489, 496, 481, 515, 510, 540, 518, 498, 494, 478, 518, 477, 496, 490, 493, 517, 504, 491, 522, 506, 498, 514, 539, 559, 509, 514, 501, 540, 472, 557, 485, 555, 500, 498, 533, 527, 492, 482, 489, 484, 537, 534, 487, 559] payload_rows_per_dst=[5568, 5656, 5480, 5578, 5464, 5550, 5526, 5530] dup_rows_per_dst=[2642, 2818, 2538, 2448, 2574, 2702, 2728, 2734] combine_rows_per_src=[5568, 5656, 5480, 5578, 5464, 5550, 5526, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159973376 dispatch_meta_bytes=229200 combine_payload_bytes=159973376 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160202576 ingress_bytes=185139200 link_bytes=185139200 token_bytes=320175952 weight_movement_bytes=25165824 total_nvlink_bytes=345341776 weight_to_token_ratio=0.078600 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408695395e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159948800 dispatch_meta_bytes=229992 combine_payload_bytes=159948800 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160178792 ingress_bytes=185114624 link_bytes=185114624 token_bytes=320127592 weight_movement_bytes=25165824 total_nvlink_bytes=345293416 weight_to_token_ratio=0.078612 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408508401e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159981568 dispatch_meta_bytes=230000 combine_payload_bytes=159981568 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160211568 ingress_bytes=210313216 link_bytes=210313216 token_bytes=320193136 weight_movement_bytes=50331648 total_nvlink_bytes=370524784 weight_to_token_ratio=0.157192 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600240570e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44380 payload_rows_remote=38824 dup_rows=21156 dup_rows_remote=18474 dedup_ratio=0.322815 router_tokens_per_expert=[551, 506, 501, 524, 494, 481, 521, 552, 479, 467, 521, 522, 518, 507, 511, 502, 499, 523, 514, 489, 541, 502, 553, 498, 524, 546, 520, 511, 520, 506, 527, 493, 554, 551, 529, 491, 524, 513, 485, 508, 569, 505, 501, 478, 490, 514, 554, 515, 485, 502, 524, 536, 528, 494, 515, 485, 481, 499, 484, 528, 496, 528, 496, 483] payload_rows_per_dst=[5604, 5506, 5556, 5670, 5620, 5590, 5434, 5400] dup_rows_per_dst=[2656, 2548, 2682, 2624, 2690, 2662, 2704, 2590] combine_rows_per_src=[5604, 5506, 5556, 5670, 5620, 5590, 5434, 5400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38926 dup_rows=21034 dup_rows_remote=18372 dedup_ratio=0.320953 router_tokens_per_expert=[477, 552, 487, 488, 481, 546, 505, 503, 486, 511, 503, 474, 535, 525, 521, 490, 495, 490, 507, 534, 512, 493, 532, 526, 510, 506, 513, 482, 503, 507, 509, 531, 500, 475, 539, 523, 498, 520, 504, 548, 499, 517, 539, 488, 523, 497, 527, 529, 577, 529, 516, 501, 517, 506, 525, 544, 498, 516, 528, 494, 522, 499, 522, 514] payload_rows_per_dst=[5500, 5598, 5590, 5524, 5530, 5576, 5708, 5476] dup_rows_per_dst=[2578, 2492, 2588, 2598, 2684, 2662, 2722, 2710] combine_rows_per_src=[5500, 5598, 5590, 5524, 5530, 5576, 5708, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44660 payload_rows_remote=39120 dup_rows=20876 dup_rows_remote=18196 dedup_ratio=0.318542 router_tokens_per_expert=[525, 527, 514, 528, 513, 491, 517, 514, 517, 522, 542, 514, 543, 541, 536, 496, 512, 499, 492, 523, 511, 502, 523, 489, 507, 500, 538, 513, 532, 513, 499, 507, 516, 523, 511, 522, 524, 516, 492, 506, 478, 516, 520, 492, 538, 478, 500, 509, 504, 484, 508, 446, 527, 557, 508, 496, 506, 536, 515, 501, 524, 520, 500, 495] payload_rows_per_dst=[5610, 5682, 5524, 5620, 5540, 5548, 5522, 5614] dup_rows_per_dst=[2648, 2740, 2578, 2598, 2680, 2514, 2538, 2580] combine_rows_per_src=[5610, 5682, 5524, 5620, 5540, 5548, 5522, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160006144 dispatch_meta_bytes=228792 combine_payload_bytes=160006144 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160234936 ingress_bytes=210337792 link_bytes=210337792 token_bytes=320241080 weight_movement_bytes=50331648 total_nvlink_bytes=370572728 weight_to_token_ratio=0.157168 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600427565e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159014912 dispatch_meta_bytes=229088 combine_payload_bytes=159014912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159244000 ingress_bytes=184180736 link_bytes=184180736 token_bytes=318258912 weight_movement_bytes=25165824 total_nvlink_bytes=343424736 weight_to_token_ratio=0.079073 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401402592e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5106811328 prefetch_bytes=327155712 reduce_bytes=352321536 weight_movement_bytes=679477248 egress_bytes=2555240896 ingress_bytes=3231047680 link_bytes=3231047680 total_nvlink_bytes=5786288576 weight_to_token_ratio=0.133053 prefetch_share_of_ingress=0.101254 dedup_ratio=0.321398 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=14 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.536533723e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159440896 dispatch_meta_bytes=229192 combine_payload_bytes=159440896 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159670088 ingress_bytes=234938368 link_bytes=234938368 token_bytes=319110984 weight_movement_bytes=75497472 total_nvlink_bytes=394608456 weight_to_token_ratio=0.236587 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787609524e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159023104 dispatch_meta_bytes=229192 combine_payload_bytes=159023104 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159252296 ingress_bytes=259686400 link_bytes=259686400 token_bytes=318275400 weight_movement_bytes=100663296 total_nvlink_bytes=418938696 weight_to_token_ratio=0.316277 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975913453e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160235520 dispatch_meta_bytes=229264 combine_payload_bytes=160235520 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160464784 ingress_bytes=185401344 link_bytes=185401344 token_bytes=320700304 weight_movement_bytes=25165824 total_nvlink_bytes=345866128 weight_to_token_ratio=0.078471 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410690008e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5092199712 prefetch_bytes=377487360 reduce_bytes=201326592 weight_movement_bytes=578813952 egress_bytes=2547928352 ingress_bytes=3123085312 link_bytes=3123085312 total_nvlink_bytes=5671013664 weight_to_token_ratio=0.113667 prefetch_share_of_ingress=0.120870 dedup_ratio=0.321600 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=8 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.485191919e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38888 dup_rows=21118 dup_rows_remote=18578 dedup_ratio=0.322235 router_tokens_per_expert=[498, 520, 505, 499, 480, 482, 522, 508, 497, 505, 515, 505, 482, 495, 505, 519, 502, 508, 513, 539, 526, 518, 529, 537, 501, 512, 510, 535, 498, 520, 505, 544, 519, 479, 529, 487, 498, 530, 524, 511, 513, 513, 529, 544, 521, 523, 536, 573, 506, 515, 516, 532, 504, 511, 513, 507, 497, 488, 481, 477, 507, 493, 509, 549] payload_rows_per_dst=[5530, 5512, 5606, 5606, 5628, 5622, 5482, 5432] dup_rows_per_dst=[2540, 2534, 2834, 2644, 2636, 2786, 2574, 2570] combine_rows_per_src=[5530, 5512, 5606, 5606, 5628, 5622, 5482, 5432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38764 dup_rows=21108 dup_rows_remote=18478 dedup_ratio=0.322083 router_tokens_per_expert=[500, 506, 517, 478, 511, 518, 536, 473, 477, 503, 524, 540, 575, 519, 504, 505, 561, 526, 523, 494, 474, 543, 508, 511, 504, 535, 503, 512, 471, 526, 504, 491, 515, 531, 509, 503, 505, 517, 503, 513, 485, 519, 474, 528, 497, 544, 496, 520, 527, 507, 495, 537, 505, 504, 524, 511, 477, 560, 502, 541, 476, 482, 529, 560] payload_rows_per_dst=[5494, 5664, 5530, 5548, 5568, 5476, 5558, 5590] dup_rows_per_dst=[2584, 2630, 2750, 2544, 2624, 2650, 2662, 2664] combine_rows_per_src=[5494, 5664, 5530, 5548, 5568, 5476, 5558, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5114727952 prefetch_bytes=352321536 reduce_bytes=301989888 weight_movement_bytes=654311424 egress_bytes=2559200784 ingress_bytes=3209838592 link_bytes=3209838592 total_nvlink_bytes=5769039376 weight_to_token_ratio=0.127927 prefetch_share_of_ingress=0.109763 dedup_ratio=0.320381 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=12 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.526447683e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5104730240 prefetch_bytes=327155712 reduce_bytes=276824064 weight_movement_bytes=603979776 egress_bytes=2554200192 ingress_bytes=3154509824 link_bytes=3154509824 total_nvlink_bytes=5708710016 weight_to_token_ratio=0.118318 prefetch_share_of_ingress=0.103710 dedup_ratio=0.321529 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=11 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.500135933e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38892 dup_rows=21028 dup_rows_remote=18312 dedup_ratio=0.320862 router_tokens_per_expert=[546, 537, 477, 516, 540, 550, 517, 517, 498, 507, 479, 494, 524, 529, 494, 506, 481, 500, 527, 514, 482, 495, 505, 509, 484, 522, 529, 523, 521, 523, 528, 536, 544, 536, 501, 519, 515, 491, 533, 520, 511, 515, 490, 495, 465, 498, 507, 482, 519, 539, 510, 491, 532, 503, 498, 515, 506, 532, 508, 533, 510, 507, 505, 528] payload_rows_per_dst=[5690, 5452, 5508, 5616, 5626, 5510, 5534, 5572] dup_rows_per_dst=[2710, 2610, 2518, 2716, 2692, 2416, 2680, 2686] combine_rows_per_src=[5690, 5452, 5508, 5616, 5626, 5510, 5534, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=229864 combine_payload_bytes=159285248 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159515112 ingress_bytes=209616896 link_bytes=209616896 token_bytes=318800360 weight_movement_bytes=50331648 total_nvlink_bytes=369132008 weight_to_token_ratio=0.157878 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594942380e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158777344 dispatch_meta_bytes=228968 combine_payload_bytes=158777344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159006312 ingress_bytes=183943168 link_bytes=183943168 token_bytes=317783656 weight_movement_bytes=25165824 total_nvlink_bytes=342949480 weight_to_token_ratio=0.079192 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399594974e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44640 payload_rows_remote=39006 dup_rows=20896 dup_rows_remote=18286 dedup_ratio=0.318848 router_tokens_per_expert=[532, 506, 510, 486, 486, 478, 514, 547, 484, 539, 500, 500, 494, 501, 498, 513, 513, 510, 542, 483, 511, 489, 534, 540, 492, 530, 514, 517, 521, 515, 534, 515, 531, 495, 469, 540, 511, 511, 505, 532, 479, 480, 546, 506, 529, 489, 542, 526, 518, 526, 525, 500, 511, 524, 495, 534, 531, 523, 481, 519, 494, 500, 513, 535] payload_rows_per_dst=[5586, 5606, 5634, 5630, 5562, 5560, 5526, 5536] dup_rows_per_dst=[2532, 2452, 2610, 2646, 2626, 2634, 2740, 2656] combine_rows_per_src=[5586, 5606, 5634, 5630, 5562, 5560, 5526, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159301632 dispatch_meta_bytes=228816 combine_payload_bytes=159301632 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159530448 ingress_bytes=184467456 link_bytes=184467456 token_bytes=318832080 weight_movement_bytes=25165824 total_nvlink_bytes=343997904 weight_to_token_ratio=0.078931 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403584200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5100018648 prefetch_bytes=301989888 reduce_bytes=553648128 weight_movement_bytes=855638016 egress_bytes=2551839704 ingress_bytes=3403816960 link_bytes=3403816960 total_nvlink_bytes=5955656664 weight_to_token_ratio=0.167772 prefetch_share_of_ingress=0.088721 dedup_ratio=0.320900 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=22 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=19812 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.618694636e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5105139888 prefetch_bytes=377487360 reduce_bytes=201326592 weight_movement_bytes=578813952 egress_bytes=2554405040 ingress_bytes=3129548800 link_bytes=3129778184 total_nvlink_bytes=5683953840 weight_to_token_ratio=0.113379 prefetch_share_of_ingress=0.120620 dedup_ratio=0.321270 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=8 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.488374733e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159768576 dispatch_meta_bytes=229168 combine_payload_bytes=159768576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159997744 ingress_bytes=184934400 link_bytes=184934400 token_bytes=319766320 weight_movement_bytes=25165824 total_nvlink_bytes=344932144 weight_to_token_ratio=0.078701 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407137104e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5107726272 prefetch_bytes=327155712 reduce_bytes=402653184 weight_movement_bytes=729808896 egress_bytes=2555697088 ingress_bytes=3281838080 link_bytes=3281838080 total_nvlink_bytes=5837535168 weight_to_token_ratio=0.142883 prefetch_share_of_ingress=0.099687 dedup_ratio=0.320694 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=16 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.560687240e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5104466288 prefetch_bytes=377487360 reduce_bytes=478150656 weight_movement_bytes=855638016 egress_bytes=2554067312 ingress_bytes=3406036992 link_bytes=3406036992 total_nvlink_bytes=5960104304 weight_to_token_ratio=0.167625 prefetch_share_of_ingress=0.110829 dedup_ratio=0.321129 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=19 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040211 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.619750378e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none + [2026-09-21 08:32:50] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 211.9 | throughput per GPU (TFLOP/s/GPU): 152.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.896814E+00 | loss scale: 1.0 | grad norm: 7.812 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=39014 dup_rows=21068 dup_rows_remote=18506 dedup_ratio=0.321472 router_tokens_per_expert=[507, 498, 500, 484, 515, 515, 497, 501, 514, 511, 537, 541, 521, 536, 530, 510, 489, 526, 484, 508, 468, 531, 523, 524, 557, 548, 521, 485, 506, 540, 491, 528, 492, 491, 503, 499, 500, 534, 490, 506, 513, 560, 500, 524, 507, 508, 545, 520, 509, 520, 530, 517, 486, 478, 471, 497, 562, 528, 463, 518, 526, 499, 510, 516] payload_rows_per_dst=[5480, 5588, 5500, 5650, 5562, 5628, 5454, 5606] dup_rows_per_dst=[2554, 2812, 2606, 2702, 2468, 2726, 2562, 2638] combine_rows_per_src=[5480, 5588, 5500, 5650, 5562, 5628, 5454, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38884 dup_rows=21098 dup_rows_remote=18450 dedup_ratio=0.321930 router_tokens_per_expert=[532, 493, 475, 498, 524, 498, 526, 543, 488, 519, 525, 532, 486, 482, 520, 519, 540, 494, 519, 483, 523, 522, 548, 531, 482, 530, 519, 474, 494, 581, 494, 506, 524, 457, 524, 532, 535, 522, 535, 523, 506, 533, 537, 489, 477, 525, 520, 514, 521, 540, 485, 505, 482, 495, 501, 544, 503, 466, 511, 486, 538, 547, 516, 475] payload_rows_per_dst=[5572, 5528, 5628, 5584, 5554, 5554, 5544, 5474] dup_rows_per_dst=[2606, 2614, 2692, 2576, 2750, 2648, 2602, 2610] combine_rows_per_src=[5572, 5528, 5628, 5584, 5554, 5554, 5544, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38918 dup_rows=21092 dup_rows_remote=18502 dedup_ratio=0.321838 router_tokens_per_expert=[506, 476, 514, 516, 516, 524, 520, 519, 502, 501, 558, 513, 540, 486, 511, 518, 459, 485, 539, 522, 512, 511, 522, 529, 520, 535, 501, 503, 485, 548, 531, 538, 517, 490, 481, 470, 482, 536, 527, 555, 481, 517, 568, 514, 507, 516, 527, 452, 486, 509, 539, 513, 476, 495, 533, 535, 515, 486, 506, 466, 526, 493, 564, 526] payload_rows_per_dst=[5568, 5680, 5486, 5592, 5526, 5488, 5550, 5554] dup_rows_per_dst=[2614, 2578, 2672, 2730, 2590, 2676, 2622, 2610] combine_rows_per_src=[5568, 5680, 5486, 5592, 5526, 5488, 5550, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44252 payload_rows_remote=38578 dup_rows=21284 dup_rows_remote=18588 dedup_ratio=0.324768 router_tokens_per_expert=[530, 525, 472, 546, 498, 492, 527, 463, 498, 524, 516, 514, 582, 481, 511, 541, 545, 522, 514, 529, 502, 448, 510, 487, 481, 484, 470, 512, 491, 495, 496, 547, 473, 524, 512, 528, 495, 493, 566, 501, 517, 492, 511, 543, 528, 519, 479, 477, 504, 500, 581, 507, 534, 529, 487, 530, 507, 539, 537, 522, 507, 529, 503, 541] payload_rows_per_dst=[5458, 5412, 5662, 5364, 5474, 5440, 5768, 5674] dup_rows_per_dst=[2648, 2472, 2902, 2588, 2640, 2500, 2838, 2696] combine_rows_per_src=[5458, 5412, 5662, 5364, 5474, 5440, 5768, 5674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38948 dup_rows=21002 dup_rows_remote=18430 dedup_ratio=0.320465 router_tokens_per_expert=[527, 533, 525, 519, 537, 534, 479, 491, 492, 540, 512, 519, 538, 495, 539, 491, 504, 517, 501, 494, 499, 515, 478, 529, 514, 511, 537, 573, 488, 472, 538, 446, 505, 516, 504, 535, 528, 512, 507, 527, 512, 524, 499, 519, 508, 537, 503, 513, 490, 522, 532, 486, 470, 560, 472, 500, 488, 528, 522, 517, 514, 504, 495, 532] payload_rows_per_dst=[5556, 5646, 5514, 5586, 5612, 5566, 5476, 5578] dup_rows_per_dst=[2734, 2606, 2560, 2572, 2656, 2664, 2588, 2622] combine_rows_per_src=[5556, 5646, 5514, 5586, 5612, 5566, 5476, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44694 payload_rows_remote=39146 dup_rows=20842 dup_rows_remote=18236 dedup_ratio=0.318024 router_tokens_per_expert=[499, 523, 485, 513, 545, 537, 510, 526, 515, 548, 519, 548, 505, 507, 527, 519, 513, 436, 519, 532, 498, 526, 536, 517, 530, 513, 507, 477, 484, 550, 516, 504, 514, 492, 513, 495, 519, 528, 515, 535, 513, 515, 504, 516, 485, 528, 516, 509, 492, 511, 479, 521, 513, 487, 490, 498, 497, 505, 520, 515, 518, 532, 510, 499] payload_rows_per_dst=[5652, 5674, 5548, 5494, 5636, 5584, 5510, 5596] dup_rows_per_dst=[2624, 2702, 2606, 2668, 2586, 2588, 2472, 2596] combine_rows_per_src=[5652, 5674, 5548, 5494, 5636, 5584, 5510, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=39058 dup_rows=20910 dup_rows_remote=18256 dedup_ratio=0.319061 router_tokens_per_expert=[495, 480, 527, 515, 527, 541, 468, 512, 509, 486, 522, 496, 532, 533, 511, 522, 525, 488, 557, 541, 499, 487, 539, 473, 498, 528, 514, 513, 479, 521, 514, 496, 505, 477, 516, 500, 503, 538, 503, 523, 507, 497, 499, 526, 526, 523, 542, 550, 546, 519, 496, 499, 529, 525, 531, 520, 505, 465, 503, 504, 510, 493, 518, 522] payload_rows_per_dst=[5552, 5568, 5618, 5564, 5544, 5612, 5660, 5508] dup_rows_per_dst=[2578, 2654, 2600, 2562, 2586, 2728, 2670, 2532] combine_rows_per_src=[5552, 5568, 5618, 5564, 5544, 5612, 5660, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=229336 combine_payload_bytes=159268864 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159498200 ingress_bytes=209600512 link_bytes=209600512 token_bytes=318767064 weight_movement_bytes=50331648 total_nvlink_bytes=369098712 weight_to_token_ratio=0.157895 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594817716e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159801344 dispatch_meta_bytes=230080 combine_payload_bytes=159801344 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160031424 ingress_bytes=184967168 link_bytes=184967168 token_bytes=319832768 weight_movement_bytes=25165824 total_nvlink_bytes=344998592 weight_to_token_ratio=0.078684 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407386431e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158015488 dispatch_meta_bytes=228664 combine_payload_bytes=158015488 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158244152 ingress_bytes=208347136 link_bytes=208347136 token_bytes=316259640 weight_movement_bytes=50331648 total_nvlink_bytes=366591288 weight_to_token_ratio=0.159147 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.585280973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=229680 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159637808 ingress_bytes=234905600 link_bytes=234905600 token_bytes=319045936 weight_movement_bytes=75497472 total_nvlink_bytes=394543408 weight_to_token_ratio=0.236635 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787360198e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44398 payload_rows_remote=38804 dup_rows=21138 dup_rows_remote=18392 dedup_ratio=0.322540 router_tokens_per_expert=[532, 503, 502, 531, 498, 466, 484, 509, 531, 498, 515, 521, 507, 516, 518, 530, 493, 506, 482, 477, 500, 516, 490, 507, 523, 510, 535, 478, 517, 544, 492, 504, 496, 521, 540, 555, 517, 530, 522, 537, 541, 460, 474, 444, 525, 543, 536, 544, 509, 521, 504, 521, 571, 478, 496, 510, 517, 558, 486, 533, 493, 483, 546, 522] payload_rows_per_dst=[5594, 5448, 5420, 5584, 5632, 5548, 5580, 5592] dup_rows_per_dst=[2746, 2554, 2522, 2720, 2720, 2586, 2640, 2650] combine_rows_per_src=[5594, 5448, 5420, 5584, 5632, 5548, 5580, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159531008 dispatch_meta_bytes=229512 combine_payload_bytes=159531008 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159760520 ingress_bytes=184696832 link_bytes=184696832 token_bytes=319291528 weight_movement_bytes=25165824 total_nvlink_bytes=344457352 weight_to_token_ratio=0.078818 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405329486e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160342016 dispatch_meta_bytes=229528 combine_payload_bytes=160342016 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160571544 ingress_bytes=185507840 link_bytes=185507840 token_bytes=320913560 weight_movement_bytes=25165824 total_nvlink_bytes=346079384 weight_to_token_ratio=0.078419 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411500320e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159981568 dispatch_meta_bytes=229256 combine_payload_bytes=159981568 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160210824 ingress_bytes=235479040 link_bytes=235479040 token_bytes=320192392 weight_movement_bytes=75497472 total_nvlink_bytes=395689864 weight_to_token_ratio=0.235788 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791723413e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44706 payload_rows_remote=39080 dup_rows=20830 dup_rows_remote=18214 dedup_ratio=0.317841 router_tokens_per_expert=[489, 501, 513, 496, 493, 491, 526, 504, 515, 493, 504, 498, 486, 492, 502, 496, 511, 501, 503, 519, 483, 471, 519, 501, 533, 531, 522, 529, 507, 544, 506, 513, 520, 518, 536, 535, 512, 533, 506, 491, 537, 532, 547, 509, 491, 506, 519, 480, 470, 550, 551, 515, 543, 509, 536, 496, 493, 518, 533, 532, 537, 512, 510, 499] payload_rows_per_dst=[5524, 5522, 5474, 5682, 5652, 5626, 5636, 5590] dup_rows_per_dst=[2502, 2450, 2542, 2688, 2650, 2616, 2704, 2678] combine_rows_per_src=[5524, 5522, 5474, 5682, 5652, 5626, 5636, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38844 dup_rows=21118 dup_rows_remote=18466 dedup_ratio=0.322235 router_tokens_per_expert=[531, 503, 513, 511, 479, 494, 483, 523, 530, 488, 502, 463, 468, 534, 495, 515, 546, 525, 571, 490, 530, 495, 531, 552, 487, 545, 522, 522, 516, 505, 541, 499, 518, 481, 524, 468, 519, 491, 503, 527, 487, 474, 519, 508, 501, 497, 525, 517, 541, 510, 492, 539, 529, 525, 502, 549, 509, 522, 525, 516, 517, 498, 494, 532] payload_rows_per_dst=[5478, 5466, 5666, 5460, 5622, 5496, 5656, 5574] dup_rows_per_dst=[2596, 2524, 2814, 2514, 2740, 2560, 2718, 2652] combine_rows_per_src=[5478, 5466, 5666, 5460, 5622, 5496, 5656, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44586 payload_rows_remote=39028 dup_rows=20950 dup_rows_remote=18252 dedup_ratio=0.319672 router_tokens_per_expert=[534, 559, 504, 542, 484, 521, 486, 529, 497, 503, 509, 543, 527, 508, 532, 507, 533, 521, 544, 513, 521, 520, 510, 559, 507, 490, 495, 505, 493, 521, 479, 530, 532, 507, 498, 492, 506, 524, 512, 500, 521, 496, 499, 516, 495, 498, 500, 509, 497, 526, 484, 506, 542, 515, 533, 525, 482, 505, 490, 535, 491, 500, 529, 477] payload_rows_per_dst=[5570, 5610, 5694, 5508, 5628, 5504, 5558, 5514] dup_rows_per_dst=[2748, 2642, 2748, 2532, 2514, 2564, 2698, 2504] combine_rows_per_src=[5570, 5610, 5694, 5508, 5628, 5504, 5558, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=228784 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159169968 ingress_bytes=184107008 link_bytes=184107008 token_bytes=318111152 weight_movement_bytes=25165824 total_nvlink_bytes=343276976 weight_to_token_ratio=0.079110 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400841607e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44492 payload_rows_remote=38872 dup_rows=21044 dup_rows_remote=18432 dedup_ratio=0.321106 router_tokens_per_expert=[509, 551, 470, 518, 476, 509, 464, 542, 511, 519, 525, 504, 516, 520, 466, 478, 502, 500, 532, 529, 524, 494, 505, 521, 521, 519, 481, 499, 536, 524, 492, 533, 500, 545, 499, 514, 500, 484, 535, 539, 504, 490, 484, 526, 517, 535, 569, 503, 492, 522, 486, 547, 514, 499, 543, 489, 533, 497, 526, 477, 529, 520, 523, 537] payload_rows_per_dst=[5526, 5536, 5588, 5514, 5620, 5544, 5568, 5596] dup_rows_per_dst=[2552, 2542, 2626, 2696, 2612, 2712, 2616, 2688] combine_rows_per_src=[5526, 5536, 5588, 5514, 5620, 5544, 5568, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44594 payload_rows_remote=38940 dup_rows=20942 dup_rows_remote=18290 dedup_ratio=0.319550 router_tokens_per_expert=[532, 515, 502, 530, 465, 527, 543, 518, 534, 470, 539, 490, 481, 517, 540, 486, 529, 496, 560, 507, 492, 494, 499, 517, 525, 506, 520, 538, 513, 508, 543, 500, 535, 489, 481, 511, 521, 512, 494, 449, 536, 506, 525, 492, 532, 522, 511, 533, 497, 529, 516, 516, 496, 509, 539, 505, 499, 515, 529, 550, 506, 500, 482, 495] payload_rows_per_dst=[5628, 5522, 5582, 5654, 5454, 5650, 5550, 5554] dup_rows_per_dst=[2636, 2592, 2606, 2652, 2530, 2664, 2664, 2598] combine_rows_per_src=[5628, 5522, 5582, 5654, 5454, 5650, 5550, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44676 payload_rows_remote=38982 dup_rows=20860 dup_rows_remote=18206 dedup_ratio=0.318298 router_tokens_per_expert=[508, 498, 523, 506, 529, 533, 543, 537, 468, 500, 548, 523, 506, 493, 496, 495, 520, 550, 516, 510, 513, 527, 486, 552, 524, 480, 467, 508, 517, 541, 534, 527, 497, 499, 508, 494, 504, 518, 491, 500, 491, 507, 550, 525, 478, 524, 507, 488, 476, 491, 532, 499, 519, 531, 527, 514, 533, 521, 525, 484, 516, 513, 506, 522] payload_rows_per_dst=[5678, 5494, 5694, 5618, 5540, 5566, 5528, 5558] dup_rows_per_dst=[2676, 2564, 2654, 2578, 2482, 2574, 2650, 2682] combine_rows_per_src=[5678, 5494, 5694, 5618, 5540, 5566, 5528, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160071680 dispatch_meta_bytes=229176 combine_payload_bytes=160071680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160300856 ingress_bytes=185237504 link_bytes=185237504 token_bytes=320372536 weight_movement_bytes=25165824 total_nvlink_bytes=345538360 weight_to_token_ratio=0.078552 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409443375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38976 dup_rows=21036 dup_rows_remote=18476 dedup_ratio=0.320984 router_tokens_per_expert=[525, 521, 513, 527, 529, 493, 498, 491, 495, 486, 521, 516, 520, 466, 515, 523, 505, 502, 521, 504, 487, 529, 527, 495, 483, 516, 493, 527, 498, 507, 527, 520, 487, 529, 528, 530, 495, 515, 500, 497, 552, 514, 505, 513, 511, 519, 534, 551, 515, 549, 492, 522, 537, 499, 486, 518, 549, 499, 513, 505, 514, 509, 500, 501] payload_rows_per_dst=[5534, 5524, 5488, 5640, 5562, 5630, 5578, 5544] dup_rows_per_dst=[2660, 2560, 2652, 2502, 2600, 2768, 2658, 2636] combine_rows_per_src=[5534, 5524, 5488, 5640, 5562, 5630, 5578, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=229240 combine_payload_bytes=159105024 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159334264 ingress_bytes=209436672 link_bytes=209436672 token_bytes=318439288 weight_movement_bytes=50331648 total_nvlink_bytes=368770936 weight_to_token_ratio=0.158057 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593571083e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159858688 dispatch_meta_bytes=229120 combine_payload_bytes=159858688 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160087808 ingress_bytes=210190336 link_bytes=210190336 token_bytes=319946496 weight_movement_bytes=50331648 total_nvlink_bytes=370278144 weight_to_token_ratio=0.157313 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599305595e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159219712 dispatch_meta_bytes=229216 combine_payload_bytes=159219712 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159448928 ingress_bytes=184385536 link_bytes=184385536 token_bytes=318668640 weight_movement_bytes=25165824 total_nvlink_bytes=343834464 weight_to_token_ratio=0.078972 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402960883e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=228920 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159727160 ingress_bytes=209829888 link_bytes=209829888 token_bytes=319225400 weight_movement_bytes=50331648 total_nvlink_bytes=369557048 weight_to_token_ratio=0.157668 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596563003e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=228752 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159899024 ingress_bytes=210001920 link_bytes=210001920 token_bytes=319569296 weight_movement_bytes=50331648 total_nvlink_bytes=369900944 weight_to_token_ratio=0.157498 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597871967e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=229808 combine_payload_bytes=159645696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159875504 ingress_bytes=184811520 link_bytes=184811520 token_bytes=319521200 weight_movement_bytes=25165824 total_nvlink_bytes=344687024 weight_to_token_ratio=0.078761 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406202129e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44316 payload_rows_remote=38676 dup_rows=21220 dup_rows_remote=18632 dedup_ratio=0.323792 router_tokens_per_expert=[513, 517, 471, 525, 528, 536, 515, 493, 501, 508, 521, 512, 506, 463, 527, 502, 511, 531, 504, 514, 492, 568, 509, 520, 522, 492, 524, 521, 500, 515, 501, 510, 507, 529, 493, 513, 540, 523, 522, 538, 496, 479, 473, 476, 520, 525, 504, 549, 505, 497, 475, 524, 526, 516, 480, 522, 512, 544, 499, 572, 527, 511, 521, 478] payload_rows_per_dst=[5640, 5828, 5334, 5596, 5600, 5432, 5338, 5548] dup_rows_per_dst=[2588, 3160, 2374, 2702, 2730, 2630, 2416, 2620] combine_rows_per_src=[5640, 5828, 5334, 5596, 5600, 5432, 5338, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44632 payload_rows_remote=39002 dup_rows=20904 dup_rows_remote=18236 dedup_ratio=0.318970 router_tokens_per_expert=[521, 523, 493, 527, 509, 508, 515, 506, 496, 487, 490, 518, 491, 491, 534, 495, 499, 532, 517, 557, 509, 497, 509, 491, 484, 530, 548, 491, 486, 513, 500, 509, 519, 538, 504, 517, 540, 565, 515, 512, 530, 519, 500, 534, 541, 517, 520, 488, 507, 505, 508, 502, 517, 581, 506, 505, 525, 509, 505, 494, 521, 480, 464, 504] payload_rows_per_dst=[5554, 5532, 5556, 5528, 5712, 5630, 5608, 5512] dup_rows_per_dst=[2650, 2472, 2666, 2594, 2708, 2668, 2654, 2492] combine_rows_per_src=[5554, 5532, 5556, 5528, 5712, 5630, 5608, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=38678 dup_rows=20982 dup_rows_remote=17698 dedup_ratio=0.320160 router_tokens_per_expert=[541, 526, 545, 542, 516, 459, 491, 483, 546, 514, 518, 548, 505, 497, 527, 483, 513, 501, 498, 529, 474, 514, 561, 564, 468, 512, 518, 529, 480, 525, 523, 490, 566, 511, 475, 546, 508, 499, 526, 492, 503, 514, 468, 540, 508, 501, 502, 495, 499, 524, 510, 513, 558, 480, 495, 514, 521, 503, 502, 509, 512, 501, 548, 485] payload_rows_per_dst=[5554, 5516, 5578, 5694, 5210, 5520, 5606, 5876] dup_rows_per_dst=[2652, 2658, 2522, 2704, 2038, 2542, 2582, 3284] combine_rows_per_src=[5554, 5516, 5578, 5694, 5210, 5520, 5606, 5876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44426 payload_rows_remote=38906 dup_rows=21110 dup_rows_remote=18480 dedup_ratio=0.322113 router_tokens_per_expert=[512, 550, 550, 509, 480, 547, 500, 543, 496, 494, 521, 539, 497, 505, 522, 516, 547, 478, 513, 510, 485, 506, 515, 488, 515, 532, 510, 493, 519, 546, 499, 531, 485, 511, 513, 522, 485, 531, 514, 503, 499, 499, 521, 539, 521, 498, 522, 525, 492, 495, 537, 515, 488, 507, 497, 544, 491, 508, 498, 513, 483, 517, 525, 502] payload_rows_per_dst=[5654, 5570, 5536, 5614, 5496, 5538, 5520, 5498] dup_rows_per_dst=[2728, 2610, 2548, 2676, 2532, 2710, 2630, 2676] combine_rows_per_src=[5654, 5570, 5536, 5614, 5496, 5538, 5520, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38840 dup_rows=21000 dup_rows_remote=18298 dedup_ratio=0.320435 router_tokens_per_expert=[524, 486, 481, 523, 464, 506, 535, 523, 520, 537, 510, 519, 542, 476, 535, 541, 472, 471, 560, 506, 496, 519, 514, 510, 491, 526, 501, 526, 515, 516, 501, 539, 545, 491, 533, 519, 532, 515, 533, 531, 523, 494, 488, 501, 510, 530, 522, 476, 513, 511, 500, 470, 531, 511, 558, 533, 519, 503, 477, 516, 519, 484, 506, 489] payload_rows_per_dst=[5540, 5640, 5480, 5544, 5696, 5530, 5582, 5524] dup_rows_per_dst=[2544, 2720, 2616, 2686, 2702, 2558, 2672, 2502] combine_rows_per_src=[5540, 5640, 5480, 5544, 5696, 5530, 5582, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158416896 dispatch_meta_bytes=229232 combine_payload_bytes=158416896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158646128 ingress_bytes=183582720 link_bytes=183582720 token_bytes=317063024 weight_movement_bytes=25165824 total_nvlink_bytes=342228848 weight_to_token_ratio=0.079372 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396852381e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44342 payload_rows_remote=38850 dup_rows=21194 dup_rows_remote=18676 dedup_ratio=0.323395 router_tokens_per_expert=[504, 516, 480, 549, 503, 465, 550, 498, 548, 500, 523, 525, 515, 531, 512, 489, 520, 503, 511, 533, 518, 528, 516, 544, 476, 499, 489, 520, 533, 532, 452, 504, 516, 523, 493, 520, 525, 524, 524, 535, 529, 508, 516, 523, 542, 548, 509, 494, 537, 539, 501, 489, 497, 499, 507, 523, 482, 464, 482, 493, 469, 511, 535, 525] payload_rows_per_dst=[5470, 5644, 5636, 5492, 5618, 5588, 5536, 5358] dup_rows_per_dst=[2660, 2642, 2710, 2518, 2702, 2750, 2648, 2564] combine_rows_per_src=[5470, 5644, 5636, 5492, 5618, 5588, 5536, 5358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159752192 dispatch_meta_bytes=228952 combine_payload_bytes=159752192 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159981144 ingress_bytes=210083840 link_bytes=210083840 token_bytes=319733336 weight_movement_bytes=50331648 total_nvlink_bytes=370064984 weight_to_token_ratio=0.157418 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598495284e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44552 payload_rows_remote=38948 dup_rows=20984 dup_rows_remote=18264 dedup_ratio=0.320190 router_tokens_per_expert=[505, 500, 506, 543, 512, 534, 539, 481, 531, 506, 512, 483, 515, 510, 526, 475, 474, 534, 501, 537, 507, 537, 545, 527, 472, 505, 543, 498, 486, 517, 507, 515, 497, 536, 506, 510, 509, 493, 528, 503, 529, 504, 526, 514, 546, 506, 512, 532, 555, 514, 490, 519, 484, 504, 518, 503, 518, 518, 486, 487, 502, 499, 518, 519] payload_rows_per_dst=[5580, 5564, 5604, 5488, 5578, 5656, 5550, 5532] dup_rows_per_dst=[2660, 2552, 2720, 2598, 2586, 2682, 2624, 2562] combine_rows_per_src=[5580, 5564, 5604, 5488, 5578, 5656, 5550, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158425088 dispatch_meta_bytes=225504 combine_payload_bytes=158425088 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158650592 ingress_bytes=183590912 link_bytes=183590912 token_bytes=317075680 weight_movement_bytes=25165824 total_nvlink_bytes=342241504 weight_to_token_ratio=0.079369 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396914713e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229544 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159588520 ingress_bytes=209690624 link_bytes=209690624 token_bytes=318947496 weight_movement_bytes=50331648 total_nvlink_bytes=369279144 weight_to_token_ratio=0.157805 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595503365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44354 payload_rows_remote=38810 dup_rows=21182 dup_rows_remote=18466 dedup_ratio=0.323212 router_tokens_per_expert=[508, 511, 514, 521, 494, 525, 507, 505, 536, 526, 529, 505, 477, 506, 506, 545, 528, 517, 475, 509, 541, 506, 512, 468, 478, 534, 466, 528, 513, 508, 484, 535, 496, 510, 516, 466, 540, 539, 517, 506, 491, 508, 514, 556, 497, 481, 521, 521, 546, 514, 536, 489, 508, 536, 518, 503, 514, 533, 497, 538, 504, 535, 518, 483] payload_rows_per_dst=[5560, 5544, 5516, 5496, 5494, 5606, 5560, 5578] dup_rows_per_dst=[2610, 2716, 2596, 2596, 2686, 2572, 2740, 2666] combine_rows_per_src=[5560, 5544, 5516, 5496, 5494, 5606, 5560, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=228552 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159317192 ingress_bytes=209420288 link_bytes=209420288 token_bytes=318405832 weight_movement_bytes=50331648 total_nvlink_bytes=368737480 weight_to_token_ratio=0.158074 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593446420e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159129600 dispatch_meta_bytes=230104 combine_payload_bytes=159129600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159359704 ingress_bytes=184295424 link_bytes=184295424 token_bytes=318489304 weight_movement_bytes=25165824 total_nvlink_bytes=343655128 weight_to_token_ratio=0.079016 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402275235e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159531008 dispatch_meta_bytes=228848 combine_payload_bytes=159531008 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159759856 ingress_bytes=209862656 link_bytes=209862656 token_bytes=319290864 weight_movement_bytes=50331648 total_nvlink_bytes=369622512 weight_to_token_ratio=0.157636 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596812329e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158965760 dispatch_meta_bytes=229104 combine_payload_bytes=158965760 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159194864 ingress_bytes=234463232 link_bytes=234463232 token_bytes=318160624 weight_movement_bytes=75497472 total_nvlink_bytes=393658096 weight_to_token_ratio=0.237294 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783994288e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44358 payload_rows_remote=38768 dup_rows=21178 dup_rows_remote=18648 dedup_ratio=0.323151 router_tokens_per_expert=[513, 487, 556, 509, 522, 496, 448, 497, 489, 509, 533, 471, 513, 562, 507, 526, 527, 513, 528, 495, 553, 517, 496, 497, 496, 537, 524, 522, 504, 523, 520, 528, 524, 514, 555, 521, 518, 538, 469, 505, 525, 550, 542, 507, 515, 535, 525, 518, 511, 511, 500, 524, 465, 504, 509, 509, 503, 474, 521, 477, 499, 500, 486, 496] payload_rows_per_dst=[5590, 5468, 5524, 5594, 5814, 5466, 5484, 5418] dup_rows_per_dst=[2530, 2688, 2728, 2714, 2964, 2520, 2540, 2494] combine_rows_per_src=[5590, 5468, 5524, 5594, 5814, 5466, 5484, 5418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44540 payload_rows_remote=38984 dup_rows=20996 dup_rows_remote=18270 dedup_ratio=0.320374 router_tokens_per_expert=[515, 488, 514, 545, 513, 521, 514, 520, 502, 483, 510, 501, 498, 516, 492, 496, 476, 523, 498, 501, 516, 490, 522, 503, 499, 531, 526, 520, 547, 471, 498, 554, 509, 513, 528, 497, 548, 507, 517, 482, 521, 508, 484, 527, 513, 504, 554, 530, 536, 511, 496, 534, 512, 495, 527, 490, 521, 486, 510, 481, 529, 510, 542, 543] payload_rows_per_dst=[5686, 5466, 5460, 5658, 5588, 5556, 5570, 5556] dup_rows_per_dst=[2574, 2530, 2598, 2634, 2614, 2726, 2632, 2688] combine_rows_per_src=[5686, 5466, 5460, 5658, 5588, 5556, 5570, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=38946 dup_rows=20968 dup_rows_remote=18346 dedup_ratio=0.319946 router_tokens_per_expert=[536, 514, 515, 510, 525, 519, 519, 534, 528, 514, 542, 515, 541, 516, 527, 492, 504, 491, 559, 485, 471, 527, 486, 485, 523, 518, 500, 525, 493, 507, 520, 546, 493, 543, 513, 467, 508, 535, 507, 502, 515, 470, 486, 557, 516, 532, 513, 534, 491, 518, 527, 485, 457, 514, 478, 506, 524, 520, 510, 532, 519, 494, 509, 506] payload_rows_per_dst=[5592, 5624, 5468, 5648, 5538, 5586, 5490, 5622] dup_rows_per_dst=[2576, 2726, 2548, 2776, 2496, 2660, 2564, 2622] combine_rows_per_src=[5592, 5624, 5468, 5648, 5538, 5586, 5490, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44424 payload_rows_remote=38844 dup_rows=21112 dup_rows_remote=18540 dedup_ratio=0.322144 router_tokens_per_expert=[531, 523, 561, 509, 504, 465, 503, 521, 530, 523, 481, 510, 524, 526, 506, 530, 501, 549, 498, 543, 506, 493, 494, 508, 520, 481, 497, 517, 471, 491, 505, 504, 534, 498, 492, 522, 498, 533, 510, 500, 517, 535, 500, 506, 479, 478, 568, 543, 481, 539, 482, 527, 501, 486, 525, 535, 518, 542, 506, 501, 529, 535, 494, 529] payload_rows_per_dst=[5504, 5608, 5518, 5470, 5546, 5616, 5580, 5582] dup_rows_per_dst=[2730, 2652, 2666, 2502, 2628, 2636, 2572, 2726] combine_rows_per_src=[5504, 5608, 5518, 5470, 5546, 5616, 5580, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=38856 dup_rows=21096 dup_rows_remote=18490 dedup_ratio=0.321899 router_tokens_per_expert=[470, 519, 486, 516, 549, 493, 478, 461, 524, 511, 508, 552, 530, 489, 535, 538, 523, 529, 540, 544, 523, 507, 528, 518, 506, 509, 504, 491, 515, 509, 471, 506, 514, 520, 521, 508, 459, 519, 504, 550, 504, 505, 487, 538, 496, 526, 530, 506, 545, 527, 516, 531, 483, 491, 494, 507, 497, 517, 507, 518, 531, 519, 527, 489] payload_rows_per_dst=[5442, 5638, 5690, 5424, 5584, 5534, 5518, 5610] dup_rows_per_dst=[2502, 2736, 2734, 2598, 2606, 2650, 2670, 2600] combine_rows_per_src=[5442, 5638, 5690, 5424, 5584, 5534, 5518, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44614 payload_rows_remote=39054 dup_rows=20922 dup_rows_remote=18234 dedup_ratio=0.319244 router_tokens_per_expert=[510, 561, 481, 517, 525, 552, 507, 536, 492, 472, 559, 513, 499, 487, 492, 503, 542, 507, 511, 502, 501, 457, 489, 490, 528, 530, 480, 562, 501, 512, 531, 480, 490, 523, 528, 509, 516, 549, 507, 501, 529, 509, 521, 508, 510, 531, 482, 534, 506, 517, 548, 504, 500, 513, 514, 526, 502, 522, 485, 529, 515, 490, 481, 540] payload_rows_per_dst=[5674, 5502, 5514, 5560, 5600, 5596, 5610, 5558] dup_rows_per_dst=[2704, 2532, 2484, 2688, 2646, 2652, 2646, 2570] combine_rows_per_src=[5674, 5502, 5514, 5560, 5600, 5596, 5610, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158793728 dispatch_meta_bytes=229664 combine_payload_bytes=158793728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159023392 ingress_bytes=183959552 link_bytes=183959552 token_bytes=317817120 weight_movement_bytes=25165824 total_nvlink_bytes=342982944 weight_to_token_ratio=0.079183 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399719637e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229168 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159751984 ingress_bytes=235020288 link_bytes=235020288 token_bytes=319274800 weight_movement_bytes=75497472 total_nvlink_bytes=394772272 weight_to_token_ratio=0.236465 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788232841e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=38942 dup_rows=20978 dup_rows_remote=18388 dedup_ratio=0.320099 router_tokens_per_expert=[497, 477, 524, 500, 506, 500, 525, 498, 521, 525, 516, 493, 516, 473, 530, 496, 522, 482, 544, 539, 494, 480, 523, 519, 504, 526, 526, 489, 488, 533, 533, 517, 493, 519, 543, 537, 499, 499, 523, 515, 504, 471, 515, 536, 567, 509, 482, 520, 523, 488, 521, 456, 527, 517, 498, 502, 543, 536, 529, 498, 490, 537, 536, 519] payload_rows_per_dst=[5498, 5552, 5616, 5570, 5602, 5546, 5526, 5648] dup_rows_per_dst=[2556, 2588, 2590, 2662, 2654, 2662, 2538, 2728] combine_rows_per_src=[5498, 5552, 5616, 5570, 5602, 5546, 5526, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159678464 dispatch_meta_bytes=229016 combine_payload_bytes=159678464 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159907480 ingress_bytes=210010112 link_bytes=210010112 token_bytes=319585944 weight_movement_bytes=50331648 total_nvlink_bytes=369917592 weight_to_token_ratio=0.157490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597934299e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=229536 combine_payload_bytes=159105024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159334560 ingress_bytes=184270848 link_bytes=184270848 token_bytes=318439584 weight_movement_bytes=25165824 total_nvlink_bytes=343605408 weight_to_token_ratio=0.079029 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402088240e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=38966 dup_rows=20990 dup_rows_remote=18502 dedup_ratio=0.320282 router_tokens_per_expert=[516, 499, 536, 514, 498, 535, 550, 522, 494, 474, 520, 496, 532, 490, 521, 507, 497, 528, 491, 498, 526, 502, 513, 499, 490, 537, 498, 493, 528, 490, 486, 518, 514, 536, 495, 520, 518, 517, 501, 508, 532, 516, 523, 538, 502, 484, 536, 504, 535, 480, 525, 479, 553, 537, 544, 480, 518, 515, 497, 515, 510, 543, 525, 470] payload_rows_per_dst=[5680, 5580, 5516, 5530, 5550, 5610, 5530, 5550] dup_rows_per_dst=[2660, 2488, 2592, 2550, 2668, 2660, 2736, 2636] combine_rows_per_src=[5680, 5580, 5516, 5530, 5550, 5610, 5530, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229384 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159383560 ingress_bytes=234651648 link_bytes=234651648 token_bytes=318537736 weight_movement_bytes=75497472 total_nvlink_bytes=394035208 weight_to_token_ratio=0.237013 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785427916e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159965184 dispatch_meta_bytes=229152 combine_payload_bytes=159965184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160194336 ingress_bytes=185131008 link_bytes=185131008 token_bytes=320159520 weight_movement_bytes=25165824 total_nvlink_bytes=345325344 weight_to_token_ratio=0.078604 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408633064e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=229320 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159735752 ingress_bytes=184672256 link_bytes=184672256 token_bytes=319242184 weight_movement_bytes=25165824 total_nvlink_bytes=344408008 weight_to_token_ratio=0.078830 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405142491e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159604736 dispatch_meta_bytes=229872 combine_payload_bytes=159604736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159834608 ingress_bytes=184770560 link_bytes=184770560 token_bytes=319439344 weight_movement_bytes=25165824 total_nvlink_bytes=344605168 weight_to_token_ratio=0.078781 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405890471e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44566 payload_rows_remote=38954 dup_rows=20970 dup_rows_remote=18424 dedup_ratio=0.319977 router_tokens_per_expert=[525, 505, 492, 475, 512, 534, 496, 540, 474, 504, 551, 520, 488, 506, 506, 521, 484, 473, 507, 538, 497, 459, 475, 492, 528, 526, 501, 497, 532, 545, 486, 518, 513, 518, 523, 544, 531, 545, 517, 481, 508, 500, 527, 524, 507, 482, 515, 542, 557, 498, 497, 550, 549, 513, 514, 499, 526, 531, 514, 507, 505, 487, 532, 505] payload_rows_per_dst=[5612, 5598, 5656, 5628, 5578, 5430, 5658, 5406] dup_rows_per_dst=[2546, 2716, 2886, 2638, 2638, 2416, 2696, 2434] combine_rows_per_src=[5612, 5598, 5656, 5628, 5578, 5430, 5658, 5406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44578 payload_rows_remote=38962 dup_rows=20958 dup_rows_remote=18328 dedup_ratio=0.319794 router_tokens_per_expert=[476, 495, 510, 522, 551, 510, 553, 511, 511, 509, 523, 492, 498, 529, 535, 520, 522, 521, 548, 553, 513, 513, 496, 483, 526, 534, 536, 509, 474, 498, 503, 521, 491, 520, 532, 471, 510, 489, 501, 494, 537, 513, 500, 499, 518, 496, 520, 501, 537, 501, 516, 496, 516, 458, 481, 553, 514, 492, 529, 511, 525, 543, 490, 519] payload_rows_per_dst=[5318, 5602, 5610, 5654, 5672, 5594, 5512, 5616] dup_rows_per_dst=[2318, 2616, 2688, 2564, 2964, 2574, 2604, 2630] combine_rows_per_src=[5318, 5602, 5610, 5654, 5672, 5594, 5512, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44358 payload_rows_remote=38870 dup_rows=21178 dup_rows_remote=18574 dedup_ratio=0.323151 router_tokens_per_expert=[502, 528, 490, 540, 507, 492, 552, 510, 519, 486, 500, 526, 521, 506, 523, 482, 508, 482, 489, 535, 491, 458, 550, 562, 523, 548, 491, 537, 499, 503, 497, 497, 514, 527, 493, 529, 507, 492, 525, 528, 520, 523, 527, 491, 556, 517, 501, 490, 504, 496, 523, 501, 524, 517, 475, 506, 495, 520, 537, 535, 536, 497, 500, 508] payload_rows_per_dst=[5552, 5512, 5536, 5552, 5518, 5572, 5488, 5628] dup_rows_per_dst=[2690, 2614, 2614, 2638, 2712, 2678, 2604, 2628] combine_rows_per_src=[5552, 5512, 5536, 5552, 5518, 5572, 5488, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44550 payload_rows_remote=38952 dup_rows=20986 dup_rows_remote=18416 dedup_ratio=0.320221 router_tokens_per_expert=[540, 507, 500, 551, 535, 519, 517, 513, 535, 526, 520, 499, 525, 499, 504, 512, 492, 538, 508, 442, 506, 501, 496, 521, 495, 538, 543, 527, 516, 531, 516, 513, 523, 514, 480, 494, 459, 531, 564, 489, 520, 523, 502, 501, 512, 502, 503, 521, 511, 497, 489, 526, 497, 463, 509, 526, 523, 510, 545, 473, 557, 515, 527, 477] payload_rows_per_dst=[5668, 5564, 5474, 5632, 5492, 5598, 5512, 5610] dup_rows_per_dst=[2696, 2676, 2534, 2726, 2616, 2570, 2524, 2644] combine_rows_per_src=[5668, 5564, 5474, 5632, 5492, 5598, 5512, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44518 payload_rows_remote=39010 dup_rows=21018 dup_rows_remote=18448 dedup_ratio=0.320709 router_tokens_per_expert=[496, 513, 480, 552, 567, 530, 533, 574, 519, 473, 499, 478, 499, 552, 521, 507, 507, 511, 525, 495, 517, 487, 476, 515, 500, 496, 496, 500, 550, 510, 490, 483, 512, 460, 531, 519, 513, 502, 495, 507, 499, 528, 496, 486, 507, 532, 523, 512, 520, 523, 559, 512, 498, 536, 480, 518, 506, 532, 549, 503, 522, 526, 515, 496] payload_rows_per_dst=[5676, 5534, 5500, 5518, 5508, 5554, 5624, 5604] dup_rows_per_dst=[2814, 2562, 2566, 2532, 2570, 2612, 2668, 2694] combine_rows_per_src=[5676, 5534, 5500, 5518, 5508, 5554, 5624, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38886 dup_rows=21120 dup_rows_remote=18474 dedup_ratio=0.322266 router_tokens_per_expert=[536, 515, 528, 505, 531, 506, 508, 522, 514, 496, 492, 524, 505, 540, 500, 538, 517, 491, 489, 494, 503, 498, 480, 490, 520, 524, 519, 508, 478, 521, 513, 505, 509, 513, 522, 493, 515, 511, 493, 513, 510, 509, 544, 507, 502, 499, 499, 505, 562, 489, 557, 527, 491, 518, 534, 511, 544, 527, 511, 512, 508, 499, 505, 519] payload_rows_per_dst=[5618, 5532, 5428, 5530, 5568, 5502, 5666, 5572] dup_rows_per_dst=[2684, 2686, 2496, 2646, 2570, 2648, 2712, 2678] combine_rows_per_src=[5618, 5532, 5428, 5530, 5568, 5502, 5666, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159588352 dispatch_meta_bytes=229160 combine_payload_bytes=159588352 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159817512 ingress_bytes=260251648 link_bytes=260251648 token_bytes=319405864 weight_movement_bytes=100663296 total_nvlink_bytes=420069160 weight_to_token_ratio=0.315158 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.980214337e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=229512 combine_payload_bytes=159555584 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159785096 ingress_bytes=209887232 link_bytes=209887232 token_bytes=319340680 weight_movement_bytes=50331648 total_nvlink_bytes=369672328 weight_to_token_ratio=0.157611 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596999324e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159211520 dispatch_meta_bytes=229776 combine_payload_bytes=159211520 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159441296 ingress_bytes=184377344 link_bytes=184377344 token_bytes=318652816 weight_movement_bytes=25165824 total_nvlink_bytes=343818640 weight_to_token_ratio=0.078976 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402898552e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159547392 dispatch_meta_bytes=229472 combine_payload_bytes=159547392 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159776864 ingress_bytes=184713216 link_bytes=184713216 token_bytes=319324256 weight_movement_bytes=25165824 total_nvlink_bytes=344490080 weight_to_token_ratio=0.078810 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405454149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44398 payload_rows_remote=38834 dup_rows=21138 dup_rows_remote=18396 dedup_ratio=0.322540 router_tokens_per_expert=[504, 533, 501, 503, 508, 495, 521, 490, 496, 536, 531, 497, 483, 491, 521, 529, 501, 500, 504, 521, 520, 530, 538, 539, 514, 510, 518, 517, 518, 516, 522, 483, 503, 512, 512, 501, 529, 535, 487, 506, 529, 551, 533, 490, 534, 487, 496, 491, 491, 518, 527, 508, 545, 534, 517, 493, 509, 494, 499, 528, 511, 547, 508, 453] payload_rows_per_dst=[5492, 5566, 5564, 5550, 5544, 5566, 5634, 5482] dup_rows_per_dst=[2618, 2602, 2742, 2646, 2626, 2656, 2632, 2616] combine_rows_per_src=[5492, 5566, 5564, 5550, 5544, 5566, 5634, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44716 payload_rows_remote=39030 dup_rows=20820 dup_rows_remote=18162 dedup_ratio=0.317688 router_tokens_per_expert=[506, 487, 513, 514, 538, 524, 487, 505, 484, 514, 508, 553, 505, 517, 537, 554, 522, 488, 495, 517, 528, 508, 512, 544, 491, 522, 478, 503, 495, 492, 525, 511, 478, 515, 500, 535, 512, 510, 540, 493, 479, 483, 504, 500, 497, 525, 518, 511, 479, 510, 475, 521, 511, 552, 533, 531, 547, 539, 534, 503, 464, 489, 514, 589] payload_rows_per_dst=[5600, 5686, 5532, 5456, 5608, 5470, 5606, 5758] dup_rows_per_dst=[2548, 2658, 2696, 2578, 2558, 2564, 2618, 2600] combine_rows_per_src=[5600, 5686, 5532, 5456, 5608, 5470, 5606, 5758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=229832 combine_payload_bytes=159784960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160014792 ingress_bytes=184950784 link_bytes=184950784 token_bytes=319799752 weight_movement_bytes=25165824 total_nvlink_bytes=344965576 weight_to_token_ratio=0.078692 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407261767e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=229440 combine_payload_bytes=159277056 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159506496 ingress_bytes=209608704 link_bytes=209608704 token_bytes=318783552 weight_movement_bytes=50331648 total_nvlink_bytes=369115200 weight_to_token_ratio=0.157887 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594880048e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159064064 dispatch_meta_bytes=228920 combine_payload_bytes=159064064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159292984 ingress_bytes=184229888 link_bytes=184229888 token_bytes=318357048 weight_movement_bytes=25165824 total_nvlink_bytes=343522872 weight_to_token_ratio=0.079049 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401776582e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=228768 combine_payload_bytes=159866880 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160095648 ingress_bytes=210198528 link_bytes=210198528 token_bytes=319962528 weight_movement_bytes=50331648 total_nvlink_bytes=370294176 weight_to_token_ratio=0.157305 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599367927e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44542 payload_rows_remote=38962 dup_rows=20994 dup_rows_remote=18388 dedup_ratio=0.320343 router_tokens_per_expert=[485, 494, 501, 509, 565, 512, 518, 529, 526, 499, 520, 518, 514, 518, 508, 567, 497, 514, 542, 510, 490, 596, 508, 534, 484, 545, 479, 497, 547, 518, 483, 478, 530, 501, 507, 525, 494, 531, 475, 486, 550, 516, 516, 520, 491, 515, 485, 483, 491, 499, 522, 501, 538, 511, 529, 500, 517, 497, 526, 501, 500, 509, 481, 516] payload_rows_per_dst=[5526, 5522, 5686, 5620, 5448, 5558, 5602, 5580] dup_rows_per_dst=[2550, 2622, 2632, 2738, 2650, 2594, 2602, 2606] combine_rows_per_src=[5526, 5522, 5686, 5620, 5448, 5558, 5602, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38808 dup_rows=21166 dup_rows_remote=18508 dedup_ratio=0.322968 router_tokens_per_expert=[519, 511, 500, 521, 514, 463, 516, 531, 523, 531, 544, 494, 527, 489, 517, 531, 461, 529, 519, 524, 505, 498, 544, 523, 542, 531, 528, 520, 486, 489, 497, 503, 507, 494, 548, 495, 462, 501, 527, 520, 513, 539, 482, 506, 506, 536, 508, 489, 522, 536, 506, 531, 524, 495, 484, 512, 535, 517, 506, 465, 546, 493, 526, 507] payload_rows_per_dst=[5508, 5632, 5582, 5548, 5502, 5512, 5562, 5524] dup_rows_per_dst=[2642, 2680, 2624, 2644, 2606, 2646, 2658, 2666] combine_rows_per_src=[5508, 5632, 5582, 5548, 5502, 5512, 5562, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44378 payload_rows_remote=38820 dup_rows=21158 dup_rows_remote=18446 dedup_ratio=0.322845 router_tokens_per_expert=[513, 526, 496, 526, 506, 516, 532, 499, 519, 518, 488, 548, 487, 540, 502, 483, 491, 488, 486, 529, 510, 490, 521, 481, 494, 511, 524, 522, 515, 493, 500, 543, 521, 523, 518, 519, 506, 477, 495, 531, 508, 510, 502, 547, 510, 524, 533, 501, 490, 530, 523, 512, 523, 521, 512, 526, 508, 504, 514, 536, 507, 540, 512, 488] payload_rows_per_dst=[5590, 5602, 5464, 5568, 5506, 5558, 5562, 5528] dup_rows_per_dst=[2638, 2568, 2528, 2636, 2674, 2712, 2712, 2690] combine_rows_per_src=[5590, 5602, 5464, 5568, 5506, 5558, 5562, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44542 payload_rows_remote=38976 dup_rows=20994 dup_rows_remote=18316 dedup_ratio=0.320343 router_tokens_per_expert=[448, 525, 484, 506, 570, 501, 522, 514, 505, 516, 522, 484, 517, 509, 521, 488, 562, 517, 497, 506, 530, 550, 520, 542, 508, 503, 538, 534, 508, 555, 544, 519, 508, 498, 502, 493, 512, 482, 502, 508, 526, 455, 505, 487, 523, 537, 491, 535, 499, 486, 505, 520, 494, 508, 523, 494, 520, 485, 524, 517, 522, 541, 511, 490] payload_rows_per_dst=[5566, 5514, 5700, 5642, 5530, 5614, 5564, 5412] dup_rows_per_dst=[2678, 2612, 2856, 2672, 2480, 2764, 2568, 2364] combine_rows_per_src=[5566, 5514, 5700, 5642, 5530, 5614, 5564, 5412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44388 payload_rows_remote=38750 dup_rows=21148 dup_rows_remote=18432 dedup_ratio=0.322693 router_tokens_per_expert=[524, 488, 479, 549, 525, 494, 502, 492, 484, 508, 515, 494, 471, 533, 514, 520, 505, 529, 519, 457, 544, 495, 509, 529, 515, 508, 530, 502, 516, 498, 497, 492, 543, 466, 559, 491, 522, 508, 521, 567, 508, 463, 525, 530, 556, 518, 533, 506, 508, 494, 578, 503, 478, 500, 552, 503, 476, 514, 528, 560, 494, 519, 521, 487] payload_rows_per_dst=[5502, 5488, 5506, 5540, 5638, 5560, 5568, 5586] dup_rows_per_dst=[2604, 2590, 2668, 2576, 2716, 2718, 2664, 2612] combine_rows_per_src=[5502, 5488, 5506, 5540, 5638, 5560, 5568, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44628 payload_rows_remote=39126 dup_rows=20908 dup_rows_remote=18322 dedup_ratio=0.319031 router_tokens_per_expert=[522, 518, 485, 543, 493, 530, 506, 467, 525, 486, 510, 512, 494, 500, 531, 521, 515, 561, 528, 518, 498, 523, 511, 544, 503, 524, 506, 508, 498, 491, 485, 529, 464, 529, 500, 480, 526, 510, 528, 510, 502, 521, 504, 515, 498, 498, 541, 495, 530, 507, 520, 546, 516, 478, 521, 490, 519, 523, 515, 540, 500, 523, 523, 511] payload_rows_per_dst=[5540, 5524, 5608, 5502, 5548, 5576, 5596, 5734] dup_rows_per_dst=[2588, 2634, 2788, 2586, 2546, 2572, 2620, 2574] combine_rows_per_src=[5540, 5524, 5608, 5502, 5548, 5576, 5596, 5734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159588352 dispatch_meta_bytes=229400 combine_payload_bytes=159588352 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159817752 ingress_bytes=184754176 link_bytes=184754176 token_bytes=319406104 weight_movement_bytes=25165824 total_nvlink_bytes=344571928 weight_to_token_ratio=0.078789 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405765808e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158957568 dispatch_meta_bytes=229264 combine_payload_bytes=158957568 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159186832 ingress_bytes=234455040 link_bytes=234455040 token_bytes=318144400 weight_movement_bytes=75497472 total_nvlink_bytes=393641872 weight_to_token_ratio=0.237306 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783931957e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=229064 combine_payload_bytes=159006720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159235784 ingress_bytes=184172544 link_bytes=184172544 token_bytes=318242504 weight_movement_bytes=25165824 total_nvlink_bytes=343408328 weight_to_token_ratio=0.079078 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401340260e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44514 payload_rows_remote=38948 dup_rows=21022 dup_rows_remote=18508 dedup_ratio=0.320770 router_tokens_per_expert=[520, 512, 517, 534, 490, 518, 520, 526, 522, 485, 493, 513, 523, 485, 507, 512, 503, 517, 484, 500, 522, 532, 517, 531, 507, 518, 518, 522, 486, 505, 505, 535, 559, 490, 536, 515, 470, 518, 533, 486, 512, 502, 469, 465, 499, 518, 485, 512, 555, 531, 522, 482, 546, 515, 526, 494, 468, 503, 520, 528, 553, 550, 500, 527] payload_rows_per_dst=[5630, 5566, 5508, 5564, 5542, 5452, 5650, 5602] dup_rows_per_dst=[2644, 2514, 2704, 2628, 2672, 2472, 2692, 2696] combine_rows_per_src=[5630, 5566, 5508, 5564, 5542, 5452, 5650, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44338 payload_rows_remote=38874 dup_rows=21198 dup_rows_remote=18626 dedup_ratio=0.323456 router_tokens_per_expert=[538, 517, 525, 522, 517, 503, 514, 518, 505, 504, 526, 494, 517, 578, 519, 522, 502, 523, 512, 515, 482, 501, 473, 510, 529, 490, 512, 481, 524, 505, 542, 504, 532, 532, 488, 488, 517, 504, 502, 549, 512, 457, 472, 503, 506, 494, 529, 518, 544, 537, 519, 510, 516, 515, 535, 521, 512, 496, 508, 525, 474, 494, 529, 506] payload_rows_per_dst=[5582, 5680, 5464, 5570, 5488, 5398, 5604, 5552] dup_rows_per_dst=[2726, 2650, 2572, 2604, 2736, 2584, 2790, 2536] combine_rows_per_src=[5582, 5680, 5464, 5570, 5488, 5398, 5604, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=229168 combine_payload_bytes=159645696 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159874864 ingress_bytes=209977344 link_bytes=209977344 token_bytes=319520560 weight_movement_bytes=50331648 total_nvlink_bytes=369852208 weight_to_token_ratio=0.157522 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597684972e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158720000 dispatch_meta_bytes=228728 combine_payload_bytes=158720000 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158948728 ingress_bytes=183885824 link_bytes=183885824 token_bytes=317668728 weight_movement_bytes=25165824 total_nvlink_bytes=342834552 weight_to_token_ratio=0.079220 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399158652e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=160260096 dispatch_meta_bytes=229792 combine_payload_bytes=160260096 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160489888 ingress_bytes=185425920 link_bytes=185425920 token_bytes=320749984 weight_movement_bytes=25165824 total_nvlink_bytes=345915808 weight_to_token_ratio=0.078459 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410877003e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159531008 dispatch_meta_bytes=229824 combine_payload_bytes=159531008 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159760832 ingress_bytes=209862656 link_bytes=209862656 token_bytes=319291840 weight_movement_bytes=50331648 total_nvlink_bytes=369623488 weight_to_token_ratio=0.157635 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596812329e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159227904 dispatch_meta_bytes=230000 combine_payload_bytes=159227904 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159457904 ingress_bytes=209559552 link_bytes=209559552 token_bytes=318685808 weight_movement_bytes=50331648 total_nvlink_bytes=369017456 weight_to_token_ratio=0.157935 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594506058e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44318 payload_rows_remote=38814 dup_rows=21218 dup_rows_remote=18528 dedup_ratio=0.323761 router_tokens_per_expert=[505, 503, 494, 544, 535, 461, 546, 537, 505, 474, 505, 497, 495, 528, 519, 518, 533, 507, 485, 512, 532, 504, 534, 495, 550, 496, 507, 506, 475, 479, 522, 542, 505, 517, 505, 514, 508, 484, 543, 528, 526, 537, 482, 511, 495, 546, 529, 524, 544, 500, 532, 514, 522, 479, 506, 510, 491, 482, 521, 528, 495, 514, 543, 488] payload_rows_per_dst=[5526, 5450, 5478, 5562, 5526, 5602, 5670, 5504] dup_rows_per_dst=[2724, 2526, 2576, 2600, 2648, 2732, 2722, 2690] combine_rows_per_src=[5526, 5450, 5478, 5562, 5526, 5602, 5670, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44678 payload_rows_remote=39080 dup_rows=20858 dup_rows_remote=18178 dedup_ratio=0.318268 router_tokens_per_expert=[506, 488, 529, 540, 502, 481, 517, 515, 515, 514, 509, 507, 526, 522, 490, 454, 509, 487, 536, 542, 539, 499, 534, 484, 481, 509, 505, 507, 535, 508, 521, 541, 501, 507, 501, 501, 540, 505, 560, 503, 547, 526, 524, 523, 511, 502, 501, 483, 494, 504, 531, 516, 545, 504, 514, 531, 518, 510, 493, 496, 531, 507, 480, 507] payload_rows_per_dst=[5610, 5508, 5674, 5636, 5522, 5594, 5598, 5536] dup_rows_per_dst=[2546, 2566, 2586, 2578, 2714, 2640, 2680, 2548] combine_rows_per_src=[5610, 5508, 5674, 5636, 5522, 5594, 5598, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44668 payload_rows_remote=39054 dup_rows=20868 dup_rows_remote=18270 dedup_ratio=0.318420 router_tokens_per_expert=[501, 498, 479, 514, 510, 527, 469, 536, 520, 502, 529, 498, 530, 551, 520, 519, 494, 497, 510, 557, 503, 517, 496, 535, 488, 534, 499, 513, 534, 503, 518, 505, 500, 542, 483, 533, 496, 486, 495, 472, 529, 539, 515, 511, 492, 506, 514, 500, 535, 501, 528, 549, 539, 472, 499, 521, 493, 522, 515, 502, 477, 543, 548, 505] payload_rows_per_dst=[5502, 5686, 5582, 5528, 5542, 5614, 5646, 5568] dup_rows_per_dst=[2566, 2652, 2636, 2660, 2472, 2598, 2642, 2642] combine_rows_per_src=[5502, 5686, 5582, 5528, 5542, 5614, 5646, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44476 payload_rows_remote=38856 dup_rows=21060 dup_rows_remote=18414 dedup_ratio=0.321350 router_tokens_per_expert=[492, 530, 512, 528, 494, 512, 560, 505, 494, 514, 531, 518, 527, 522, 531, 495, 473, 539, 479, 539, 518, 506, 517, 530, 524, 526, 497, 488, 502, 530, 490, 549, 539, 517, 509, 510, 522, 485, 498, 515, 483, 539, 517, 513, 507, 500, 493, 485, 542, 533, 524, 501, 493, 501, 512, 533, 510, 499, 497, 480, 515, 524, 511, 489] payload_rows_per_dst=[5620, 5610, 5546, 5536, 5616, 5684, 5324, 5540] dup_rows_per_dst=[2646, 2654, 2656, 2676, 2648, 2940, 2330, 2510] combine_rows_per_src=[5620, 5610, 5546, 5536, 5616, 5684, 5324, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44556 payload_rows_remote=38974 dup_rows=20980 dup_rows_remote=18348 dedup_ratio=0.320129 router_tokens_per_expert=[511, 516, 523, 475, 499, 511, 509, 514, 523, 540, 486, 512, 514, 508, 531, 503, 512, 500, 505, 482, 525, 518, 553, 514, 498, 508, 506, 523, 504, 494, 525, 533, 504, 532, 493, 540, 497, 495, 525, 521, 513, 466, 529, 545, 498, 509, 507, 475, 510, 486, 487, 541, 509, 515, 534, 524, 497, 528, 500, 497, 524, 556, 526, 510] payload_rows_per_dst=[5538, 5570, 5600, 5550, 5582, 5556, 5542, 5618] dup_rows_per_dst=[2578, 2664, 2618, 2632, 2632, 2528, 2670, 2658] combine_rows_per_src=[5538, 5570, 5600, 5550, 5582, 5556, 5542, 5618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=39074 dup_rows=20910 dup_rows_remote=18340 dedup_ratio=0.319061 router_tokens_per_expert=[499, 552, 465, 500, 509, 487, 535, 522, 515, 548, 503, 539, 532, 505, 520, 488, 496, 494, 531, 512, 516, 512, 519, 491, 548, 513, 497, 502, 512, 491, 496, 502, 552, 500, 485, 484, 523, 515, 494, 512, 529, 504, 526, 504, 512, 515, 510, 514, 518, 509, 539, 499, 515, 509, 523, 537, 497, 499, 516, 534, 527, 491, 512, 513] payload_rows_per_dst=[5546, 5638, 5590, 5552, 5520, 5560, 5654, 5566] dup_rows_per_dst=[2592, 2662, 2552, 2570, 2610, 2668, 2644, 2612] combine_rows_per_src=[5546, 5638, 5590, 5552, 5520, 5560, 5654, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158982144 dispatch_meta_bytes=229368 combine_payload_bytes=158982144 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159211512 ingress_bytes=184147968 link_bytes=184147968 token_bytes=318193656 weight_movement_bytes=25165824 total_nvlink_bytes=343359480 weight_to_token_ratio=0.079090 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401153265e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160071680 dispatch_meta_bytes=229032 combine_payload_bytes=160071680 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160300712 ingress_bytes=210403328 link_bytes=210403328 token_bytes=320372392 weight_movement_bytes=50331648 total_nvlink_bytes=370704040 weight_to_token_ratio=0.157104 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600926218e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159965184 dispatch_meta_bytes=229296 combine_payload_bytes=159965184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160194480 ingress_bytes=185131008 link_bytes=185131008 token_bytes=320159664 weight_movement_bytes=25165824 total_nvlink_bytes=345325488 weight_to_token_ratio=0.078604 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408633064e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44238 payload_rows_remote=38730 dup_rows=21298 dup_rows_remote=18564 dedup_ratio=0.324982 router_tokens_per_expert=[511, 519, 478, 487, 530, 516, 538, 539, 505, 508, 540, 521, 513, 488, 511, 535, 522, 520, 514, 495, 545, 498, 542, 481, 541, 539, 496, 497, 466, 549, 518, 499, 528, 541, 505, 516, 524, 520, 510, 519, 493, 548, 504, 508, 495, 485, 502, 499, 479, 502, 487, 502, 480, 472, 502, 560, 536, 528, 531, 498, 517, 499, 519, 498] payload_rows_per_dst=[5558, 5508, 5646, 5528, 5610, 5382, 5444, 5562] dup_rows_per_dst=[2678, 2734, 2588, 2682, 2716, 2686, 2524, 2690] combine_rows_per_src=[5558, 5508, 5646, 5528, 5610, 5382, 5444, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229080 combine_payload_bytes=159154176 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159383256 ingress_bytes=234651648 link_bytes=234651648 token_bytes=318537432 weight_movement_bytes=75497472 total_nvlink_bytes=394034904 weight_to_token_ratio=0.237013 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.785427916e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44324 payload_rows_remote=38802 dup_rows=21212 dup_rows_remote=18586 dedup_ratio=0.323669 router_tokens_per_expert=[517, 516, 505, 488, 516, 518, 503, 485, 512, 495, 542, 532, 477, 515, 486, 511, 522, 517, 509, 519, 502, 507, 486, 512, 508, 514, 485, 518, 535, 537, 525, 524, 524, 473, 570, 483, 515, 508, 525, 501, 502, 525, 488, 510, 513, 522, 516, 532, 483, 535, 531, 531, 491, 511, 531, 512, 517, 515, 515, 532, 527, 504, 480, 508] payload_rows_per_dst=[5494, 5560, 5522, 5632, 5562, 5502, 5524, 5528] dup_rows_per_dst=[2602, 2580, 2626, 2660, 2636, 2714, 2726, 2668] combine_rows_per_src=[5494, 5560, 5522, 5632, 5562, 5502, 5524, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160047104 dispatch_meta_bytes=229656 combine_payload_bytes=160047104 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160276760 ingress_bytes=185212928 link_bytes=185212928 token_bytes=320323864 weight_movement_bytes=25165824 total_nvlink_bytes=345489688 weight_to_token_ratio=0.078564 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409256380e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159637504 dispatch_meta_bytes=229288 combine_payload_bytes=159637504 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159866792 ingress_bytes=209969152 link_bytes=209969152 token_bytes=319504296 weight_movement_bytes=50331648 total_nvlink_bytes=369835944 weight_to_token_ratio=0.157530 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597622641e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158638080 dispatch_meta_bytes=229176 combine_payload_bytes=158638080 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=158867256 ingress_bytes=259301376 link_bytes=259301376 token_bytes=317505336 weight_movement_bytes=100663296 total_nvlink_bytes=418168632 weight_to_token_ratio=0.317044 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.972983865e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158932992 dispatch_meta_bytes=229552 combine_payload_bytes=158932992 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159162544 ingress_bytes=209264640 link_bytes=209264640 token_bytes=318095536 weight_movement_bytes=50331648 total_nvlink_bytes=368427184 weight_to_token_ratio=0.158228 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592262118e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44580 payload_rows_remote=38878 dup_rows=20956 dup_rows_remote=18192 dedup_ratio=0.319763 router_tokens_per_expert=[493, 530, 533, 514, 487, 521, 530, 554, 523, 511, 542, 505, 513, 521, 537, 505, 479, 505, 531, 493, 523, 516, 518, 540, 513, 498, 504, 497, 536, 515, 515, 500, 531, 503, 517, 505, 509, 525, 469, 535, 496, 510, 541, 541, 515, 488, 479, 489, 530, 516, 489, 507, 501, 524, 521, 489, 534, 501, 474, 499, 509, 498, 506, 515] payload_rows_per_dst=[5140, 5688, 5610, 5636, 5608, 5642, 5554, 5702] dup_rows_per_dst=[2238, 2798, 2566, 2702, 2614, 2674, 2600, 2764] combine_rows_per_src=[5140, 5688, 5610, 5636, 5608, 5642, 5554, 5702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44540 payload_rows_remote=39032 dup_rows=20996 dup_rows_remote=18444 dedup_ratio=0.320374 router_tokens_per_expert=[527, 506, 537, 494, 535, 515, 510, 525, 534, 503, 523, 498, 489, 535, 500, 523, 526, 518, 482, 538, 514, 519, 555, 488, 492, 508, 507, 524, 442, 496, 504, 542, 515, 518, 491, 505, 565, 559, 482, 495, 534, 470, 510, 494, 495, 504, 509, 514, 530, 524, 533, 510, 544, 520, 529, 497, 508, 489, 499, 496, 486, 500, 489, 545] payload_rows_per_dst=[5630, 5580, 5612, 5530, 5626, 5508, 5582, 5472] dup_rows_per_dst=[2668, 2630, 2668, 2500, 2634, 2552, 2792, 2552] combine_rows_per_src=[5630, 5580, 5612, 5530, 5626, 5508, 5582, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39088 dup_rows=20934 dup_rows_remote=18404 dedup_ratio=0.319427 router_tokens_per_expert=[501, 542, 536, 518, 546, 510, 513, 542, 537, 515, 513, 526, 504, 493, 507, 542, 487, 513, 499, 491, 540, 520, 559, 502, 529, 495, 532, 543, 528, 484, 519, 496, 518, 474, 509, 500, 526, 507, 471, 503, 487, 531, 498, 510, 538, 529, 477, 527, 511, 498, 491, 503, 521, 502, 488, 508, 506, 502, 506, 517, 498, 502, 533, 495] payload_rows_per_dst=[5700, 5594, 5590, 5552, 5518, 5558, 5514, 5576] dup_rows_per_dst=[2716, 2680, 2632, 2700, 2498, 2636, 2530, 2542] combine_rows_per_src=[5700, 5594, 5590, 5552, 5518, 5558, 5514, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38878 dup_rows=21034 dup_rows_remote=18182 dedup_ratio=0.320953 router_tokens_per_expert=[501, 509, 505, 519, 523, 487, 482, 492, 514, 514, 479, 514, 522, 513, 534, 524, 475, 514, 510, 554, 522, 555, 484, 517, 508, 478, 530, 516, 490, 495, 510, 556, 497, 540, 508, 515, 505, 510, 530, 519, 503, 518, 498, 491, 536, 489, 534, 511, 490, 540, 491, 507, 511, 566, 514, 526, 512, 490, 510, 510, 470, 524, 538, 519] payload_rows_per_dst=[5624, 5562, 5554, 5586, 5600, 5562, 5482, 5532] dup_rows_per_dst=[2852, 2670, 2708, 2580, 2644, 2598, 2368, 2614] combine_rows_per_src=[5624, 5562, 5554, 5586, 5600, 5562, 5482, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44528 payload_rows_remote=39006 dup_rows=21008 dup_rows_remote=18408 dedup_ratio=0.320557 router_tokens_per_expert=[478, 492, 525, 534, 556, 528, 507, 493, 512, 536, 486, 537, 522, 531, 498, 510, 526, 490, 493, 507, 505, 520, 492, 502, 487, 524, 498, 513, 534, 535, 469, 501, 531, 535, 530, 514, 492, 507, 490, 513, 503, 499, 518, 534, 537, 516, 495, 541, 505, 510, 524, 508, 521, 461, 490, 495, 489, 533, 552, 523, 516, 508, 537, 500] payload_rows_per_dst=[5612, 5564, 5504, 5522, 5602, 5578, 5566, 5580] dup_rows_per_dst=[2614, 2700, 2566, 2600, 2622, 2708, 2462, 2736] combine_rows_per_src=[5612, 5564, 5504, 5522, 5602, 5578, 5566, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=228280 combine_payload_bytes=159244288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159472568 ingress_bytes=184410112 link_bytes=184410112 token_bytes=318716856 weight_movement_bytes=25165824 total_nvlink_bytes=343882680 weight_to_token_ratio=0.078960 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403147878e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44632 payload_rows_remote=39094 dup_rows=20904 dup_rows_remote=18318 dedup_ratio=0.318970 router_tokens_per_expert=[481, 503, 502, 536, 553, 505, 524, 501, 498, 497, 515, 505, 521, 485, 510, 492, 529, 488, 507, 541, 546, 560, 463, 526, 496, 505, 500, 503, 539, 486, 539, 490, 519, 522, 501, 482, 515, 506, 502, 515, 536, 499, 514, 499, 518, 545, 479, 523, 561, 500, 546, 494, 536, 470, 526, 517, 532, 510, 522, 521, 530, 487, 452, 543] payload_rows_per_dst=[5588, 5570, 5604, 5542, 5538, 5632, 5586, 5572] dup_rows_per_dst=[2622, 2476, 2716, 2574, 2586, 2594, 2714, 2622] combine_rows_per_src=[5588, 5570, 5604, 5542, 5538, 5632, 5586, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159875072 dispatch_meta_bytes=229904 combine_payload_bytes=159875072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160104976 ingress_bytes=185040896 link_bytes=185040896 token_bytes=319980048 weight_movement_bytes=25165824 total_nvlink_bytes=345145872 weight_to_token_ratio=0.078648 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407947416e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160104448 dispatch_meta_bytes=229968 combine_payload_bytes=160104448 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160334416 ingress_bytes=185270272 link_bytes=185270272 token_bytes=320438864 weight_movement_bytes=25165824 total_nvlink_bytes=345604688 weight_to_token_ratio=0.078535 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409692702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38926 dup_rows=21122 dup_rows_remote=18602 dedup_ratio=0.322296 router_tokens_per_expert=[497, 495, 518, 525, 490, 507, 545, 543, 484, 511, 485, 510, 480, 473, 530, 531, 512, 494, 505, 523, 535, 486, 510, 503, 563, 507, 503, 566, 521, 514, 526, 509, 516, 518, 546, 481, 500, 515, 533, 510, 505, 535, 487, 518, 500, 498, 517, 523, 551, 501, 490, 498, 520, 546, 517, 512, 513, 505, 466, 521, 507, 507, 530, 481] payload_rows_per_dst=[5644, 5488, 5450, 5664, 5540, 5544, 5622, 5462] dup_rows_per_dst=[2596, 2520, 2686, 2754, 2698, 2622, 2648, 2598] combine_rows_per_src=[5644, 5488, 5450, 5664, 5540, 5544, 5622, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44624 payload_rows_remote=39036 dup_rows=20912 dup_rows_remote=18332 dedup_ratio=0.319092 router_tokens_per_expert=[537, 514, 535, 515, 523, 522, 526, 511, 504, 497, 517, 521, 544, 498, 497, 532, 513, 491, 516, 496, 499, 521, 515, 533, 491, 526, 512, 558, 482, 478, 498, 507, 507, 478, 497, 524, 504, 497, 511, 531, 514, 521, 515, 507, 485, 507, 513, 512, 535, 546, 510, 507, 481, 517, 492, 545, 521, 486, 525, 561, 549, 509, 447, 485] payload_rows_per_dst=[5590, 5638, 5588, 5558, 5470, 5558, 5634, 5588] dup_rows_per_dst=[2776, 2582, 2580, 2546, 2628, 2590, 2632, 2578] combine_rows_per_src=[5590, 5638, 5588, 5558, 5470, 5558, 5634, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=228240 combine_payload_bytes=159244288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159472528 ingress_bytes=184410112 link_bytes=184410112 token_bytes=318716816 weight_movement_bytes=25165824 total_nvlink_bytes=343882640 weight_to_token_ratio=0.078960 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403147878e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159768576 dispatch_meta_bytes=229656 combine_payload_bytes=159768576 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159998232 ingress_bytes=184934400 link_bytes=184934400 token_bytes=319766808 weight_movement_bytes=25165824 total_nvlink_bytes=344932632 weight_to_token_ratio=0.078701 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407137104e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160129024 dispatch_meta_bytes=229648 combine_payload_bytes=160129024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160358672 ingress_bytes=185294848 link_bytes=185294848 token_bytes=320487696 weight_movement_bytes=25165824 total_nvlink_bytes=345653520 weight_to_token_ratio=0.078524 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409879697e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159440896 dispatch_meta_bytes=230112 combine_payload_bytes=159440896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159671008 ingress_bytes=184606720 link_bytes=184606720 token_bytes=319111904 weight_movement_bytes=25165824 total_nvlink_bytes=344277728 weight_to_token_ratio=0.078862 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404643838e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159891456 dispatch_meta_bytes=229472 combine_payload_bytes=159891456 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160120928 ingress_bytes=235388928 link_bytes=235388928 token_bytes=320012384 weight_movement_bytes=75497472 total_nvlink_bytes=395509856 weight_to_token_ratio=0.235920 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791037765e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38696 dup_rows=21068 dup_rows_remote=18358 dedup_ratio=0.321472 router_tokens_per_expert=[518, 524, 527, 523, 506, 525, 562, 467, 532, 547, 531, 514, 527, 523, 497, 538, 496, 527, 522, 508, 510, 495, 508, 534, 471, 521, 530, 511, 536, 485, 522, 482, 509, 522, 495, 484, 510, 516, 502, 505, 507, 497, 492, 509, 497, 492, 485, 505, 549, 492, 488, 544, 544, 524, 545, 466, 521, 497, 490, 504, 534, 521, 516, 487] payload_rows_per_dst=[5290, 5442, 5528, 5698, 5516, 5630, 5592, 5772] dup_rows_per_dst=[2262, 2580, 2538, 2962, 2570, 2734, 2712, 2710] combine_rows_per_src=[5290, 5442, 5528, 5698, 5516, 5630, 5592, 5772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38906 dup_rows=21028 dup_rows_remote=18304 dedup_ratio=0.320862 router_tokens_per_expert=[513, 495, 550, 506, 506, 495, 523, 514, 493, 523, 502, 505, 512, 503, 510, 513, 510, 533, 523, 486, 512, 502, 510, 519, 516, 518, 520, 534, 515, 519, 499, 474, 523, 487, 541, 525, 473, 491, 515, 510, 509, 515, 551, 506, 536, 527, 519, 500, 519, 498, 509, 517, 522, 525, 528, 519, 481, 481, 527, 512, 511, 501, 520, 517] payload_rows_per_dst=[5650, 5522, 5610, 5540, 5564, 5602, 5552, 5468] dup_rows_per_dst=[2554, 2600, 2580, 2650, 2566, 2724, 2722, 2632] combine_rows_per_src=[5650, 5522, 5610, 5540, 5564, 5602, 5552, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=38908 dup_rows=21096 dup_rows_remote=18460 dedup_ratio=0.321899 router_tokens_per_expert=[527, 507, 503, 519, 515, 503, 507, 534, 490, 507, 532, 538, 517, 529, 516, 504, 531, 476, 509, 504, 528, 528, 543, 534, 480, 478, 478, 485, 487, 496, 509, 543, 541, 506, 492, 510, 559, 530, 495, 506, 524, 520, 505, 501, 486, 511, 515, 489, 513, 507, 514, 513, 496, 509, 525, 507, 533, 516, 518, 513, 514, 505, 505, 533] payload_rows_per_dst=[5678, 5548, 5630, 5478, 5518, 5490, 5532, 5566] dup_rows_per_dst=[2552, 2718, 2676, 2434, 2760, 2612, 2636, 2708] combine_rows_per_src=[5678, 5548, 5630, 5478, 5518, 5490, 5532, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158498816 dispatch_meta_bytes=228216 combine_payload_bytes=158498816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158727032 ingress_bytes=183664640 link_bytes=183664640 token_bytes=317225848 weight_movement_bytes=25165824 total_nvlink_bytes=342391672 weight_to_token_ratio=0.079331 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397475698e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44336 payload_rows_remote=38712 dup_rows=21200 dup_rows_remote=18422 dedup_ratio=0.323486 router_tokens_per_expert=[537, 524, 537, 546, 518, 518, 523, 477, 536, 522, 489, 488, 493, 522, 476, 488, 533, 491, 506, 541, 504, 514, 527, 486, 525, 511, 513, 507, 513, 529, 481, 499, 536, 498, 544, 515, 500, 541, 518, 525, 559, 506, 458, 523, 504, 500, 517, 503, 502, 496, 558, 485, 510, 491, 493, 516, 511, 502, 523, 525, 469, 530, 507, 529] payload_rows_per_dst=[5624, 5446, 5498, 5512, 5644, 5558, 5544, 5510] dup_rows_per_dst=[2778, 2582, 2666, 2644, 2710, 2582, 2556, 2682] combine_rows_per_src=[5624, 5446, 5498, 5512, 5644, 5558, 5544, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38936 dup_rows=21102 dup_rows_remote=18578 dedup_ratio=0.321991 router_tokens_per_expert=[562, 507, 536, 521, 562, 496, 501, 523, 538, 517, 513, 544, 561, 509, 531, 509, 495, 517, 522, 513, 546, 500, 469, 498, 517, 491, 494, 512, 495, 472, 533, 497, 518, 541, 499, 501, 542, 528, 522, 540, 523, 494, 523, 520, 473, 507, 457, 506, 479, 528, 488, 480, 519, 492, 514, 502, 519, 520, 527, 503, 499, 497, 501, 505] payload_rows_per_dst=[5684, 5652, 5514, 5498, 5598, 5456, 5448, 5584] dup_rows_per_dst=[2732, 2792, 2606, 2524, 2784, 2550, 2556, 2558] combine_rows_per_src=[5684, 5652, 5514, 5498, 5598, 5456, 5448, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44312 payload_rows_remote=38760 dup_rows=21224 dup_rows_remote=18572 dedup_ratio=0.323853 router_tokens_per_expert=[522, 539, 514, 531, 495, 519, 526, 516, 458, 529, 516, 526, 503, 515, 541, 514, 478, 498, 496, 550, 529, 514, 462, 484, 547, 493, 502, 512, 499, 466, 510, 485, 522, 505, 518, 519, 472, 516, 502, 503, 539, 530, 507, 538, 530, 497, 535, 527, 530, 528, 492, 555, 513, 505, 522, 504, 490, 509, 534, 522, 485, 514, 495, 521] payload_rows_per_dst=[5628, 5552, 5462, 5522, 5556, 5532, 5566, 5494] dup_rows_per_dst=[2696, 2652, 2560, 2506, 2558, 2874, 2732, 2646] combine_rows_per_src=[5628, 5552, 5462, 5522, 5556, 5532, 5566, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44594 payload_rows_remote=39096 dup_rows=20942 dup_rows_remote=18372 dedup_ratio=0.319550 router_tokens_per_expert=[531, 504, 515, 480, 518, 553, 511, 510, 498, 489, 463, 524, 473, 472, 525, 513, 525, 506, 527, 545, 517, 531, 529, 493, 495, 555, 511, 495, 547, 540, 462, 537, 549, 552, 488, 484, 477, 508, 498, 478, 529, 514, 517, 509, 488, 515, 494, 528, 517, 479, 518, 521, 548, 494, 522, 512, 522, 516, 510, 491, 536, 500, 553, 507] payload_rows_per_dst=[5652, 5472, 5632, 5634, 5498, 5512, 5566, 5628] dup_rows_per_dst=[2592, 2442, 2714, 2650, 2570, 2676, 2656, 2642] combine_rows_per_src=[5652, 5472, 5632, 5634, 5498, 5512, 5566, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=228840 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159587816 ingress_bytes=184524800 link_bytes=184524800 token_bytes=318946792 weight_movement_bytes=25165824 total_nvlink_bytes=344112616 weight_to_token_ratio=0.078903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404020521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159367168 dispatch_meta_bytes=229472 combine_payload_bytes=159367168 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159596640 ingress_bytes=184532992 link_bytes=184532992 token_bytes=318963808 weight_movement_bytes=25165824 total_nvlink_bytes=344129632 weight_to_token_ratio=0.078899 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404082853e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38852 dup_rows=21098 dup_rows_remote=18416 dedup_ratio=0.321930 router_tokens_per_expert=[513, 495, 508, 517, 532, 487, 518, 524, 524, 484, 526, 540, 497, 505, 545, 493, 556, 496, 527, 527, 520, 515, 499, 494, 505, 507, 528, 494, 503, 486, 520, 550, 583, 508, 557, 515, 506, 516, 506, 475, 506, 554, 499, 536, 482, 479, 506, 506, 501, 496, 502, 484, 522, 521, 504, 493, 519, 521, 514, 490, 507, 525, 513, 487] payload_rows_per_dst=[5522, 5566, 5586, 5560, 5594, 5572, 5476, 5562] dup_rows_per_dst=[2666, 2662, 2682, 2626, 2738, 2564, 2570, 2590] combine_rows_per_src=[5522, 5566, 5586, 5560, 5594, 5572, 5476, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158564352 dispatch_meta_bytes=228536 combine_payload_bytes=158564352 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=158792888 ingress_bytes=284393472 link_bytes=284393472 token_bytes=317357240 weight_movement_bytes=125829120 total_nvlink_bytes=443186360 weight_to_token_ratio=0.396490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.163905723e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159481856 dispatch_meta_bytes=230056 combine_payload_bytes=159481856 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159711912 ingress_bytes=184647680 link_bytes=184647680 token_bytes=319193768 weight_movement_bytes=25165824 total_nvlink_bytes=344359592 weight_to_token_ratio=0.078842 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4096 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404955496e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=229328 combine_payload_bytes=158760960 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158990288 ingress_bytes=234258432 link_bytes=234258432 token_bytes=317751248 weight_movement_bytes=75497472 total_nvlink_bytes=393248720 weight_to_token_ratio=0.237599 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782435997e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160137216 dispatch_meta_bytes=229872 combine_payload_bytes=160137216 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160367088 ingress_bytes=210468864 link_bytes=210468864 token_bytes=320504304 weight_movement_bytes=50331648 total_nvlink_bytes=370835952 weight_to_token_ratio=0.157039 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601424872e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44476 payload_rows_remote=38746 dup_rows=21060 dup_rows_remote=18258 dedup_ratio=0.321350 router_tokens_per_expert=[503, 522, 532, 495, 530, 554, 502, 524, 507, 504, 513, 538, 504, 511, 542, 509, 464, 513, 543, 522, 552, 508, 510, 496, 545, 530, 498, 475, 497, 551, 516, 521, 486, 532, 495, 517, 519, 520, 505, 516, 509, 527, 504, 482, 477, 511, 474, 510, 513, 509, 512, 483, 506, 529, 518, 515, 508, 496, 494, 502, 485, 531, 519, 533] payload_rows_per_dst=[5576, 5614, 5350, 5584, 5542, 5486, 5594, 5730] dup_rows_per_dst=[2748, 2642, 2470, 2620, 2522, 2564, 2692, 2802] combine_rows_per_src=[5576, 5614, 5350, 5584, 5542, 5486, 5594, 5730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159137792 dispatch_meta_bytes=229072 combine_payload_bytes=159137792 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159366864 ingress_bytes=209469440 link_bytes=209469440 token_bytes=318504656 weight_movement_bytes=50331648 total_nvlink_bytes=368836304 weight_to_token_ratio=0.158025 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593820410e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44594 payload_rows_remote=39024 dup_rows=20942 dup_rows_remote=18326 dedup_ratio=0.319550 router_tokens_per_expert=[486, 512, 488, 505, 550, 497, 480, 487, 550, 472, 468, 514, 506, 513, 497, 502, 517, 511, 507, 542, 537, 529, 524, 511, 518, 494, 491, 539, 549, 477, 484, 474, 489, 544, 530, 496, 541, 513, 510, 513, 517, 516, 493, 503, 520, 484, 534, 526, 475, 478, 526, 554, 517, 543, 509, 538, 536, 494, 545, 543, 521, 492, 518, 519] payload_rows_per_dst=[5510, 5492, 5660, 5520, 5588, 5570, 5610, 5644] dup_rows_per_dst=[2500, 2552, 2696, 2532, 2684, 2616, 2670, 2692] combine_rows_per_src=[5510, 5492, 5660, 5520, 5588, 5570, 5610, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158703616 dispatch_meta_bytes=228016 combine_payload_bytes=158703616 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158931632 ingress_bytes=183869440 link_bytes=183869440 token_bytes=317635248 weight_movement_bytes=25165824 total_nvlink_bytes=342801072 weight_to_token_ratio=0.079229 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399033989e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44610 payload_rows_remote=39026 dup_rows=20926 dup_rows_remote=18316 dedup_ratio=0.319305 router_tokens_per_expert=[505, 528, 496, 514, 502, 508, 500, 502, 470, 489, 519, 511, 518, 521, 502, 531, 507, 546, 510, 518, 512, 525, 528, 529, 504, 547, 506, 456, 544, 522, 508, 495, 546, 492, 516, 490, 505, 468, 530, 515, 495, 541, 556, 486, 531, 513, 528, 497, 518, 528, 484, 523, 491, 543, 498, 512, 518, 515, 516, 524, 482, 505, 500, 529] payload_rows_per_dst=[5508, 5544, 5646, 5538, 5542, 5664, 5584, 5584] dup_rows_per_dst=[2602, 2578, 2704, 2626, 2582, 2630, 2610, 2594] combine_rows_per_src=[5508, 5544, 5646, 5538, 5542, 5664, 5584, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159842304 dispatch_meta_bytes=229400 combine_payload_bytes=159842304 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160071704 ingress_bytes=210173952 link_bytes=210173952 token_bytes=319914008 weight_movement_bytes=50331648 total_nvlink_bytes=370245656 weight_to_token_ratio=0.157329 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599180932e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44492 payload_rows_remote=38766 dup_rows=21044 dup_rows_remote=18392 dedup_ratio=0.321106 router_tokens_per_expert=[487, 571, 501, 492, 507, 545, 520, 518, 520, 493, 569, 501, 559, 520, 521, 509, 507, 526, 534, 487, 519, 521, 529, 486, 514, 494, 518, 535, 505, 508, 540, 513, 522, 517, 489, 530, 511, 517, 533, 468, 491, 515, 490, 513, 501, 499, 499, 496, 496, 507, 513, 493, 518, 557, 533, 503, 486, 518, 496, 479, 522, 468, 492, 527] payload_rows_per_dst=[5726, 5598, 5546, 5574, 5522, 5522, 5552, 5452] dup_rows_per_dst=[2652, 2738, 2672, 2680, 2624, 2486, 2668, 2524] combine_rows_per_src=[5726, 5598, 5546, 5574, 5522, 5522, 5552, 5452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44530 payload_rows_remote=38920 dup_rows=21006 dup_rows_remote=18254 dedup_ratio=0.320526 router_tokens_per_expert=[493, 522, 521, 485, 560, 459, 497, 498, 478, 509, 505, 474, 511, 539, 492, 541, 521, 513, 502, 539, 518, 494, 484, 513, 518, 558, 509, 560, 539, 498, 495, 504, 531, 528, 487, 537, 527, 527, 536, 514, 514, 532, 534, 526, 539, 520, 542, 542, 474, 493, 462, 504, 504, 510, 494, 530, 463, 526, 518, 486, 527, 481, 508, 503] payload_rows_per_dst=[5538, 5482, 5606, 5610, 5608, 5680, 5544, 5462] dup_rows_per_dst=[2532, 2616, 2562, 2752, 2766, 2818, 2398, 2562] combine_rows_per_src=[5538, 5482, 5606, 5610, 5608, 5680, 5544, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44296 payload_rows_remote=38832 dup_rows=21240 dup_rows_remote=18686 dedup_ratio=0.324097 router_tokens_per_expert=[486, 489, 506, 521, 516, 524, 502, 521, 488, 463, 499, 498, 500, 535, 522, 504, 512, 496, 509, 510, 511, 525, 506, 502, 493, 532, 484, 496, 526, 504, 559, 503, 536, 529, 497, 508, 571, 511, 486, 493, 542, 519, 531, 531, 528, 482, 520, 483, 496, 497, 512, 507, 549, 549, 554, 521, 488, 519, 492, 519, 512, 545, 483, 516] payload_rows_per_dst=[5570, 5464, 5500, 5552, 5594, 5518, 5612, 5486] dup_rows_per_dst=[2560, 2554, 2642, 2642, 2668, 2754, 2758, 2662] combine_rows_per_src=[5570, 5464, 5500, 5552, 5594, 5518, 5612, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38906 dup_rows=21026 dup_rows_remote=18396 dedup_ratio=0.320831 router_tokens_per_expert=[513, 536, 523, 488, 508, 499, 541, 469, 555, 527, 518, 486, 491, 524, 530, 467, 509, 525, 497, 531, 531, 513, 522, 542, 486, 474, 547, 541, 558, 505, 495, 501, 489, 488, 524, 529, 562, 513, 515, 497, 509, 507, 518, 548, 508, 495, 498, 552, 525, 503, 496, 482, 514, 495, 523, 499, 485, 525, 486, 508, 508, 514, 499, 502] payload_rows_per_dst=[5482, 5608, 5600, 5576, 5604, 5618, 5496, 5526] dup_rows_per_dst=[2672, 2588, 2740, 2638, 2630, 2652, 2578, 2528] combine_rows_per_src=[5482, 5608, 5600, 5576, 5604, 5618, 5496, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159850496 dispatch_meta_bytes=229368 combine_payload_bytes=159850496 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160079864 ingress_bytes=185016320 link_bytes=185016320 token_bytes=319930360 weight_movement_bytes=25165824 total_nvlink_bytes=345096184 weight_to_token_ratio=0.078660 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407760421e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38942 dup_rows=21028 dup_rows_remote=18386 dedup_ratio=0.320862 router_tokens_per_expert=[499, 535, 511, 508, 543, 518, 511, 517, 491, 529, 523, 490, 462, 478, 537, 481, 522, 500, 526, 509, 489, 505, 500, 553, 531, 539, 523, 511, 534, 512, 532, 502, 513, 528, 496, 506, 490, 496, 525, 503, 529, 502, 521, 489, 539, 450, 530, 521, 500, 522, 495, 511, 499, 509, 548, 570, 506, 473, 481, 506, 530, 519, 489, 551] payload_rows_per_dst=[5604, 5490, 5566, 5640, 5548, 5510, 5624, 5526] dup_rows_per_dst=[2680, 2492, 2642, 2728, 2566, 2652, 2684, 2584] combine_rows_per_src=[5604, 5490, 5566, 5640, 5548, 5510, 5624, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158785536 dispatch_meta_bytes=228632 combine_payload_bytes=158785536 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159014168 ingress_bytes=234283008 link_bytes=234283008 token_bytes=317799704 weight_movement_bytes=75497472 total_nvlink_bytes=393297176 weight_to_token_ratio=0.237563 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782622992e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=228696 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159645016 ingress_bytes=260079616 link_bytes=260079616 token_bytes=319061336 weight_movement_bytes=100663296 total_nvlink_bytes=419724632 weight_to_token_ratio=0.315498 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978905372e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44366 payload_rows_remote=38760 dup_rows=21170 dup_rows_remote=18338 dedup_ratio=0.323029 router_tokens_per_expert=[500, 500, 501, 517, 506, 471, 504, 557, 532, 516, 468, 529, 495, 525, 515, 520, 520, 514, 522, 477, 484, 480, 507, 520, 519, 480, 518, 521, 521, 512, 524, 505, 543, 567, 497, 498, 543, 516, 504, 528, 490, 522, 524, 498, 486, 515, 473, 532, 491, 469, 532, 487, 511, 523, 522, 517, 525, 545, 527, 518, 517, 516, 528, 524] payload_rows_per_dst=[5538, 5292, 5616, 5530, 5700, 5502, 5582, 5606] dup_rows_per_dst=[2574, 2360, 2980, 2670, 2692, 2508, 2554, 2832] combine_rows_per_src=[5538, 5292, 5616, 5530, 5700, 5502, 5582, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159055872 dispatch_meta_bytes=230072 combine_payload_bytes=159055872 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159285944 ingress_bytes=184221696 link_bytes=184221696 token_bytes=318341816 weight_movement_bytes=25165824 total_nvlink_bytes=343507640 weight_to_token_ratio=0.079053 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401714250e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229208 combine_payload_bytes=159358976 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159588184 ingress_bytes=209690624 link_bytes=209690624 token_bytes=318947160 weight_movement_bytes=50331648 total_nvlink_bytes=369278808 weight_to_token_ratio=0.157806 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595503365e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=229312 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159735744 ingress_bytes=209838080 link_bytes=209838080 token_bytes=319242176 weight_movement_bytes=50331648 total_nvlink_bytes=369573824 weight_to_token_ratio=0.157660 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596625334e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38832 dup_rows=21052 dup_rows_remote=18344 dedup_ratio=0.321228 router_tokens_per_expert=[519, 500, 498, 508, 497, 494, 507, 461, 537, 528, 519, 544, 492, 506, 521, 500, 486, 514, 481, 496, 517, 505, 510, 514, 525, 518, 526, 511, 524, 537, 552, 484, 524, 473, 506, 532, 521, 498, 515, 524, 546, 526, 536, 520, 508, 506, 507, 531, 518, 482, 507, 548, 495, 547, 492, 529, 516, 499, 489, 502, 503, 518, 523, 496] payload_rows_per_dst=[5464, 5536, 5496, 5608, 5588, 5652, 5614, 5526] dup_rows_per_dst=[2504, 2758, 2550, 2746, 2598, 2708, 2622, 2566] combine_rows_per_src=[5464, 5536, 5496, 5608, 5588, 5652, 5614, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=228392 combine_payload_bytes=158760960 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158989352 ingress_bytes=183926784 link_bytes=183926784 token_bytes=317750312 weight_movement_bytes=25165824 total_nvlink_bytes=342916136 weight_to_token_ratio=0.079200 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399470311e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44544 payload_rows_remote=39078 dup_rows=20992 dup_rows_remote=18306 dedup_ratio=0.320312 router_tokens_per_expert=[516, 494, 523, 492, 515, 503, 469, 544, 522, 531, 504, 502, 539, 520, 518, 496, 503, 484, 525, 474, 528, 509, 530, 512, 522, 513, 484, 538, 491, 534, 517, 500, 534, 477, 504, 508, 495, 521, 512, 540, 484, 530, 526, 574, 502, 535, 555, 497, 474, 510, 502, 490, 539, 526, 526, 509, 485, 493, 559, 497, 490, 529, 507, 486] payload_rows_per_dst=[5552, 5672, 5610, 5558, 5576, 5626, 5466, 5484] dup_rows_per_dst=[2560, 2592, 2520, 2640, 2606, 2780, 2686, 2608] combine_rows_per_src=[5552, 5672, 5610, 5558, 5576, 5626, 5466, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159055872 dispatch_meta_bytes=228704 combine_payload_bytes=159055872 prefetch_bytes=0 reduce_bytes=125829120 egress_bytes=159284576 ingress_bytes=284884992 link_bytes=284884992 token_bytes=318340448 weight_movement_bytes=125829120 total_nvlink_bytes=444169568 weight_to_token_ratio=0.395266 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.167645623e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44552 payload_rows_remote=38906 dup_rows=20984 dup_rows_remote=18176 dedup_ratio=0.320190 router_tokens_per_expert=[527, 535, 523, 503, 511, 500, 525, 521, 529, 535, 483, 512, 555, 546, 519, 487, 474, 508, 523, 520, 471, 500, 503, 533, 485, 525, 496, 509, 476, 535, 554, 523, 513, 500, 520, 506, 509, 535, 475, 500, 540, 486, 512, 532, 521, 494, 513, 548, 498, 524, 494, 531, 535, 455, 498, 506, 510, 518, 530, 517, 488, 512, 488, 514] payload_rows_per_dst=[5646, 5826, 5634, 5434, 5548, 5412, 5476, 5576] dup_rows_per_dst=[2808, 2994, 2518, 2544, 2544, 2392, 2606, 2578] combine_rows_per_src=[5646, 5826, 5634, 5434, 5548, 5412, 5476, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38832 dup_rows=21102 dup_rows_remote=18516 dedup_ratio=0.321991 router_tokens_per_expert=[492, 496, 509, 558, 529, 526, 508, 473, 501, 524, 492, 473, 517, 542, 490, 513, 485, 502, 483, 492, 488, 500, 516, 499, 517, 526, 539, 506, 506, 491, 490, 519, 506, 549, 530, 502, 519, 507, 511, 500, 529, 537, 528, 518, 532, 488, 522, 482, 494, 486, 553, 495, 528, 529, 542, 531, 494, 523, 535, 549, 499, 512, 506, 530] payload_rows_per_dst=[5566, 5522, 5424, 5602, 5598, 5552, 5626, 5544] dup_rows_per_dst=[2616, 2582, 2506, 2586, 2650, 2720, 2690, 2752] combine_rows_per_src=[5566, 5522, 5424, 5602, 5598, 5552, 5626, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38980 dup_rows=21002 dup_rows_remote=18272 dedup_ratio=0.320465 router_tokens_per_expert=[525, 495, 497, 494, 536, 489, 519, 523, 545, 504, 515, 517, 537, 531, 499, 494, 517, 494, 507, 480, 493, 499, 519, 520, 504, 496, 527, 526, 473, 495, 487, 517, 502, 513, 515, 485, 505, 535, 533, 532, 529, 531, 534, 521, 493, 493, 511, 519, 540, 529, 507, 509, 509, 490, 536, 527, 510, 491, 530, 504, 521, 495, 494, 551] payload_rows_per_dst=[5530, 5554, 5488, 5538, 5568, 5584, 5652, 5620] dup_rows_per_dst=[2626, 2730, 2570, 2512, 2672, 2678, 2642, 2572] combine_rows_per_src=[5530, 5554, 5488, 5538, 5568, 5584, 5652, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38844 dup_rows=21058 dup_rows_remote=18460 dedup_ratio=0.321320 router_tokens_per_expert=[487, 518, 547, 468, 520, 487, 527, 519, 537, 502, 520, 529, 532, 507, 546, 521, 506, 495, 539, 480, 516, 507, 517, 526, 497, 528, 539, 528, 512, 487, 557, 468, 466, 533, 504, 545, 532, 525, 538, 473, 488, 494, 501, 514, 502, 521, 515, 536, 516, 497, 484, 539, 543, 500, 491, 514, 495, 523, 512, 510, 494, 506, 504, 484] payload_rows_per_dst=[5516, 5614, 5580, 5566, 5634, 5554, 5478, 5536] dup_rows_per_dst=[2630, 2774, 2592, 2666, 2598, 2588, 2690, 2520] combine_rows_per_src=[5516, 5614, 5580, 5566, 5634, 5554, 5478, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=229536 combine_payload_bytes=160063488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160293024 ingress_bytes=185229312 link_bytes=185229312 token_bytes=320356512 weight_movement_bytes=25165824 total_nvlink_bytes=345522336 weight_to_token_ratio=0.078556 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409381044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44364 payload_rows_remote=38814 dup_rows=21172 dup_rows_remote=18384 dedup_ratio=0.323059 router_tokens_per_expert=[531, 497, 491, 521, 544, 503, 489, 500, 532, 525, 504, 502, 495, 494, 501, 508, 510, 505, 546, 503, 530, 520, 535, 520, 527, 471, 521, 535, 460, 506, 489, 499, 547, 502, 548, 509, 493, 515, 534, 499, 498, 530, 557, 501, 508, 532, 487, 494, 486, 522, 546, 491, 510, 477, 555, 483, 522, 502, 571, 500, 510, 520, 494, 511] payload_rows_per_dst=[5560, 5522, 5550, 5472, 5632, 5572, 5506, 5550] dup_rows_per_dst=[2592, 2600, 2788, 2544, 2662, 2642, 2634, 2710] combine_rows_per_src=[5560, 5522, 5550, 5472, 5632, 5572, 5506, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=228328 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159587304 ingress_bytes=184524800 link_bytes=184524800 token_bytes=318946280 weight_movement_bytes=25165824 total_nvlink_bytes=344112104 weight_to_token_ratio=0.078903 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404020521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44286 payload_rows_remote=38692 dup_rows=21250 dup_rows_remote=18510 dedup_ratio=0.324249 router_tokens_per_expert=[490, 514, 533, 519, 495, 555, 506, 510, 524, 527, 506, 535, 528, 479, 554, 522, 502, 498, 500, 515, 523, 498, 508, 508, 505, 505, 516, 525, 504, 499, 508, 490, 497, 526, 512, 495, 502, 532, 508, 534, 508, 548, 467, 509, 502, 554, 508, 506, 527, 493, 508, 520, 518, 469, 501, 502, 519, 536, 496, 502, 486, 506, 514, 562] payload_rows_per_dst=[5472, 5578, 5472, 5492, 5648, 5530, 5500, 5594] dup_rows_per_dst=[2626, 2730, 2422, 2612, 2718, 2680, 2722, 2740] combine_rows_per_src=[5472, 5578, 5472, 5492, 5648, 5530, 5500, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159055872 dispatch_meta_bytes=229392 combine_payload_bytes=159055872 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159285264 ingress_bytes=184221696 link_bytes=184221696 token_bytes=318341136 weight_movement_bytes=25165824 total_nvlink_bytes=343506960 weight_to_token_ratio=0.079053 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401714250e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159662080 dispatch_meta_bytes=229008 combine_payload_bytes=159662080 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159891088 ingress_bytes=209993728 link_bytes=209993728 token_bytes=319553168 weight_movement_bytes=50331648 total_nvlink_bytes=369884816 weight_to_token_ratio=0.157506 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597809636e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=229216 combine_payload_bytes=159105024 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159334240 ingress_bytes=209436672 link_bytes=209436672 token_bytes=318439264 weight_movement_bytes=50331648 total_nvlink_bytes=368770912 weight_to_token_ratio=0.158057 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593571083e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158982144 dispatch_meta_bytes=228792 combine_payload_bytes=158982144 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159210936 ingress_bytes=184147968 link_bytes=184147968 token_bytes=318193080 weight_movement_bytes=25165824 total_nvlink_bytes=343358904 weight_to_token_ratio=0.079090 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401153265e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44636 payload_rows_remote=39070 dup_rows=20900 dup_rows_remote=18260 dedup_ratio=0.318909 router_tokens_per_expert=[514, 531, 500, 518, 475, 537, 500, 528, 521, 488, 509, 494, 524, 539, 531, 515, 501, 507, 522, 516, 505, 547, 505, 525, 507, 534, 489, 501, 491, 547, 497, 491, 538, 528, 510, 521, 467, 520, 534, 506, 496, 529, 527, 520, 528, 479, 496, 528, 506, 474, 515, 511, 528, 518, 529, 484, 510, 497, 501, 528, 510, 493, 497, 531] payload_rows_per_dst=[5644, 5544, 5640, 5560, 5616, 5566, 5500, 5566] dup_rows_per_dst=[2562, 2698, 2616, 2554, 2632, 2640, 2630, 2568] combine_rows_per_src=[5644, 5544, 5640, 5560, 5616, 5566, 5500, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158482432 dispatch_meta_bytes=228808 combine_payload_bytes=158482432 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158711240 ingress_bytes=183648256 link_bytes=183648256 token_bytes=317193672 weight_movement_bytes=25165824 total_nvlink_bytes=342359496 weight_to_token_ratio=0.079339 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397351034e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44704 payload_rows_remote=39092 dup_rows=20832 dup_rows_remote=18320 dedup_ratio=0.317871 router_tokens_per_expert=[499, 516, 483, 491, 518, 499, 502, 509, 542, 477, 529, 482, 537, 495, 548, 478, 525, 519, 473, 499, 497, 541, 519, 477, 540, 505, 506, 496, 501, 501, 518, 476, 515, 501, 519, 491, 534, 499, 505, 485, 524, 519, 547, 503, 490, 535, 593, 535, 492, 509, 522, 496, 496, 524, 521, 502, 517, 513, 535, 572, 551, 520, 503, 502] payload_rows_per_dst=[5504, 5572, 5452, 5578, 5570, 5728, 5612, 5688] dup_rows_per_dst=[2530, 2604, 2648, 2508, 2528, 2764, 2512, 2738] combine_rows_per_src=[5504, 5572, 5452, 5578, 5570, 5728, 5612, 5688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160030720 dispatch_meta_bytes=229320 combine_payload_bytes=160030720 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160260040 ingress_bytes=235528192 link_bytes=235528192 token_bytes=320290760 weight_movement_bytes=75497472 total_nvlink_bytes=395788232 weight_to_token_ratio=0.235715 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792097403e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44320 payload_rows_remote=38584 dup_rows=21216 dup_rows_remote=18244 dedup_ratio=0.323730 router_tokens_per_expert=[495, 485, 552, 488, 520, 536, 478, 518, 515, 538, 469, 512, 524, 531, 533, 514, 542, 527, 499, 485, 520, 518, 468, 574, 502, 523, 451, 520, 502, 520, 550, 541, 484, 512, 520, 551, 544, 538, 508, 473, 516, 539, 465, 537, 494, 498, 534, 503, 556, 491, 503, 501, 527, 508, 476, 485, 513, 512, 530, 496, 469, 530, 514, 491] payload_rows_per_dst=[5736, 5638, 5574, 5540, 5438, 5340, 5512, 5542] dup_rows_per_dst=[2972, 2790, 2692, 2758, 2324, 2530, 2582, 2568] combine_rows_per_src=[5736, 5638, 5574, 5540, 5438, 5340, 5512, 5542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44196 payload_rows_remote=38660 dup_rows=21340 dup_rows_remote=18736 dedup_ratio=0.325623 router_tokens_per_expert=[561, 540, 533, 521, 561, 502, 477, 544, 485, 530, 506, 530, 500, 469, 501, 524, 516, 493, 499, 512, 523, 516, 521, 509, 542, 535, 523, 498, 492, 485, 515, 480, 505, 500, 496, 536, 530, 526, 480, 453, 519, 506, 511, 476, 503, 526, 519, 542, 507, 505, 465, 574, 532, 534, 545, 462, 505, 492, 514, 502, 538, 484, 531, 507] payload_rows_per_dst=[5700, 5512, 5546, 5536, 5442, 5476, 5526, 5458] dup_rows_per_dst=[2778, 2578, 2632, 2604, 2610, 2728, 2722, 2688] combine_rows_per_src=[5700, 5512, 5546, 5536, 5442, 5476, 5526, 5458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38910 dup_rows=21082 dup_rows_remote=18530 dedup_ratio=0.321686 router_tokens_per_expert=[487, 510, 487, 509, 480, 526, 506, 491, 545, 524, 492, 508, 483, 510, 485, 501, 508, 560, 509, 470, 521, 537, 537, 496, 475, 503, 520, 538, 538, 507, 528, 521, 548, 535, 508, 558, 504, 498, 522, 525, 493, 477, 467, 578, 512, 518, 510, 506, 536, 522, 511, 501, 522, 507, 547, 503, 489, 499, 500, 516, 524, 504, 503, 513] payload_rows_per_dst=[5480, 5544, 5600, 5588, 5642, 5496, 5528, 5576] dup_rows_per_dst=[2512, 2552, 2676, 2672, 2754, 2626, 2770, 2520] combine_rows_per_src=[5480, 5544, 5600, 5588, 5642, 5496, 5528, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160120832 dispatch_meta_bytes=229648 combine_payload_bytes=160120832 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160350480 ingress_bytes=185286656 link_bytes=185286656 token_bytes=320471312 weight_movement_bytes=25165824 total_nvlink_bytes=345637136 weight_to_token_ratio=0.078528 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409817365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44492 payload_rows_remote=38920 dup_rows=21044 dup_rows_remote=18378 dedup_ratio=0.321106 router_tokens_per_expert=[518, 490, 517, 517, 521, 504, 506, 493, 485, 511, 501, 550, 493, 515, 521, 485, 535, 516, 509, 540, 528, 530, 514, 525, 553, 490, 505, 509, 516, 505, 464, 490, 504, 478, 523, 542, 517, 512, 541, 502, 514, 517, 541, 544, 482, 489, 474, 531, 526, 548, 504, 532, 521, 489, 515, 486, 532, 474, 518, 516, 509, 521, 466, 544] payload_rows_per_dst=[5522, 5498, 5688, 5464, 5572, 5590, 5606, 5552] dup_rows_per_dst=[2610, 2624, 2706, 2600, 2666, 2594, 2636, 2608] combine_rows_per_src=[5522, 5498, 5688, 5464, 5572, 5590, 5606, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38812 dup_rows=21114 dup_rows_remote=18494 dedup_ratio=0.322174 router_tokens_per_expert=[497, 525, 511, 545, 553, 558, 529, 486, 535, 517, 570, 521, 508, 507, 487, 486, 541, 508, 514, 508, 507, 477, 526, 534, 537, 525, 511, 515, 499, 507, 513, 494, 493, 511, 529, 472, 547, 499, 531, 503, 532, 536, 476, 514, 482, 480, 512, 518, 498, 490, 499, 503, 503, 533, 488, 514, 529, 487, 515, 513, 546, 483, 486, 495] payload_rows_per_dst=[5632, 5570, 5610, 5508, 5558, 5470, 5528, 5546] dup_rows_per_dst=[2776, 2692, 2620, 2694, 2612, 2630, 2528, 2562] combine_rows_per_src=[5632, 5570, 5610, 5508, 5558, 5470, 5528, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158040064 dispatch_meta_bytes=227312 combine_payload_bytes=158040064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158267376 ingress_bytes=183205888 link_bytes=183205888 token_bytes=316307440 weight_movement_bytes=25165824 total_nvlink_bytes=341473264 weight_to_token_ratio=0.079561 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.393985125e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44378 payload_rows_remote=38878 dup_rows=21158 dup_rows_remote=18506 dedup_ratio=0.322845 router_tokens_per_expert=[511, 522, 501, 535, 520, 512, 512, 511, 525, 522, 491, 529, 510, 480, 505, 490, 492, 515, 506, 531, 482, 496, 561, 516, 539, 520, 501, 540, 529, 510, 489, 503, 542, 567, 472, 527, 514, 559, 455, 499, 521, 544, 534, 488, 530, 523, 512, 504, 502, 516, 556, 512, 523, 495, 458, 480, 521, 483, 522, 508, 520, 478, 510, 487] payload_rows_per_dst=[5524, 5508, 5580, 5452, 5532, 5652, 5630, 5500] dup_rows_per_dst=[2442, 2596, 2586, 2496, 2676, 2936, 2774, 2652] combine_rows_per_src=[5524, 5508, 5580, 5452, 5532, 5652, 5630, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158351360 dispatch_meta_bytes=229584 combine_payload_bytes=158351360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158580944 ingress_bytes=183517184 link_bytes=183517184 token_bytes=316932304 weight_movement_bytes=25165824 total_nvlink_bytes=342098128 weight_to_token_ratio=0.079404 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396353728e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159375360 dispatch_meta_bytes=229760 combine_payload_bytes=159375360 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159605120 ingress_bytes=209707008 link_bytes=209707008 token_bytes=318980480 weight_movement_bytes=50331648 total_nvlink_bytes=369312128 weight_to_token_ratio=0.157789 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595628028e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=229192 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159645512 ingress_bytes=184582144 link_bytes=184582144 token_bytes=319061832 weight_movement_bytes=25165824 total_nvlink_bytes=344227656 weight_to_token_ratio=0.078874 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404456843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158973952 dispatch_meta_bytes=229224 combine_payload_bytes=158973952 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159203176 ingress_bytes=234471424 link_bytes=234471424 token_bytes=318177128 weight_movement_bytes=75497472 total_nvlink_bytes=393674600 weight_to_token_ratio=0.237281 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784056620e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44358 payload_rows_remote=38920 dup_rows=21178 dup_rows_remote=18532 dedup_ratio=0.323151 router_tokens_per_expert=[496, 478, 536, 554, 534, 537, 518, 518, 537, 526, 517, 563, 477, 500, 485, 514, 534, 511, 548, 529, 506, 505, 503, 554, 505, 532, 479, 491, 528, 545, 508, 496, 521, 483, 515, 510, 482, 514, 524, 510, 526, 502, 491, 485, 502, 525, 503, 508, 526, 481, 488, 532, 513, 528, 497, 502, 536, 521, 480, 508, 496, 471, 504, 520] payload_rows_per_dst=[5624, 5610, 5618, 5556, 5502, 5438, 5518, 5492] dup_rows_per_dst=[2718, 2628, 2762, 2612, 2616, 2646, 2616, 2580] combine_rows_per_src=[5624, 5610, 5618, 5556, 5502, 5438, 5518, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=229536 combine_payload_bytes=159244288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159473824 ingress_bytes=184410112 link_bytes=184410112 token_bytes=318718112 weight_movement_bytes=25165824 total_nvlink_bytes=343883936 weight_to_token_ratio=0.078960 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403147878e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44506 payload_rows_remote=38980 dup_rows=21030 dup_rows_remote=18352 dedup_ratio=0.320892 router_tokens_per_expert=[552, 517, 504, 523, 522, 491, 538, 465, 532, 521, 485, 479, 503, 506, 520, 498, 537, 496, 543, 518, 515, 496, 545, 513, 537, 542, 487, 487, 535, 493, 509, 515, 529, 528, 503, 485, 541, 517, 522, 491, 488, 515, 502, 499, 529, 496, 511, 499, 515, 527, 477, 497, 526, 518, 497, 545, 479, 499, 540, 537, 541, 492, 512, 487] payload_rows_per_dst=[5610, 5516, 5626, 5586, 5578, 5544, 5526, 5520] dup_rows_per_dst=[2614, 2572, 2700, 2624, 2654, 2534, 2678, 2654] combine_rows_per_src=[5610, 5516, 5626, 5586, 5578, 5544, 5526, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=229808 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159646128 ingress_bytes=184582144 link_bytes=184582144 token_bytes=319062448 weight_movement_bytes=25165824 total_nvlink_bytes=344228272 weight_to_token_ratio=0.078874 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404456843e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44456 payload_rows_remote=38910 dup_rows=21080 dup_rows_remote=18438 dedup_ratio=0.321655 router_tokens_per_expert=[505, 520, 497, 539, 473, 504, 518, 509, 511, 547, 450, 518, 553, 536, 513, 492, 498, 520, 522, 498, 506, 514, 550, 496, 525, 527, 535, 534, 493, 510, 518, 541, 506, 471, 512, 497, 504, 484, 488, 522, 511, 545, 466, 498, 530, 562, 517, 509, 517, 502, 518, 500, 529, 517, 504, 521, 496, 507, 511, 478, 524, 505, 524, 521] payload_rows_per_dst=[5546, 5586, 5556, 5672, 5408, 5538, 5602, 5548] dup_rows_per_dst=[2642, 2654, 2652, 2894, 2560, 2480, 2614, 2584] combine_rows_per_src=[5546, 5586, 5556, 5672, 5408, 5538, 5602, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38904 dup_rows=21028 dup_rows_remote=18306 dedup_ratio=0.320862 router_tokens_per_expert=[493, 539, 528, 522, 516, 498, 511, 504, 535, 463, 531, 517, 515, 503, 532, 530, 494, 495, 507, 487, 522, 505, 485, 501, 555, 512, 494, 520, 482, 503, 541, 556, 505, 525, 508, 489, 517, 515, 534, 518, 485, 511, 525, 539, 524, 492, 521, 538, 458, 475, 558, 543, 511, 552, 467, 524, 513, 497, 507, 495, 501, 513, 510, 502] payload_rows_per_dst=[5598, 5546, 5494, 5604, 5594, 5596, 5554, 5522] dup_rows_per_dst=[2624, 2706, 2498, 2722, 2628, 2674, 2622, 2554] combine_rows_per_src=[5598, 5546, 5494, 5604, 5594, 5596, 5554, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159662080 dispatch_meta_bytes=229328 combine_payload_bytes=159662080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159891408 ingress_bytes=184827904 link_bytes=184827904 token_bytes=319553488 weight_movement_bytes=25165824 total_nvlink_bytes=344719312 weight_to_token_ratio=0.078753 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406326793e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44242 payload_rows_remote=38748 dup_rows=21294 dup_rows_remote=18738 dedup_ratio=0.324921 router_tokens_per_expert=[508, 483, 557, 496, 496, 501, 498, 514, 518, 488, 500, 472, 497, 550, 540, 519, 522, 547, 503, 538, 493, 497, 543, 539, 521, 504, 519, 535, 502, 520, 505, 505, 518, 503, 520, 499, 506, 492, 480, 507, 498, 512, 481, 523, 524, 483, 486, 502, 528, 523, 494, 526, 494, 548, 494, 513, 533, 542, 524, 541, 508, 506, 524, 506] payload_rows_per_dst=[5516, 5510, 5620, 5562, 5494, 5490, 5542, 5508] dup_rows_per_dst=[2590, 2658, 2744, 2660, 2556, 2528, 2698, 2860] combine_rows_per_src=[5516, 5510, 5620, 5562, 5494, 5490, 5542, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38896 dup_rows=21072 dup_rows_remote=18424 dedup_ratio=0.321533 router_tokens_per_expert=[514, 520, 531, 503, 547, 512, 518, 509, 525, 488, 474, 527, 513, 526, 530, 525, 527, 526, 491, 494, 489, 511, 488, 537, 520, 545, 476, 510, 518, 503, 510, 506, 522, 517, 528, 525, 571, 512, 498, 505, 525, 503, 507, 473, 532, 516, 512, 486, 488, 504, 499, 522, 531, 487, 504, 493, 551, 491, 510, 532, 500, 535, 493, 483] payload_rows_per_dst=[5596, 5568, 5510, 5504, 5674, 5540, 5520, 5552] dup_rows_per_dst=[2712, 2648, 2616, 2672, 2682, 2568, 2536, 2638] combine_rows_per_src=[5596, 5568, 5510, 5504, 5674, 5540, 5520, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44206 payload_rows_remote=38646 dup_rows=21330 dup_rows_remote=18728 dedup_ratio=0.325470 router_tokens_per_expert=[512, 491, 507, 511, 523, 537, 532, 523, 477, 509, 503, 501, 518, 529, 511, 522, 509, 523, 493, 486, 477, 545, 517, 531, 523, 509, 528, 512, 477, 493, 500, 529, 505, 483, 530, 525, 504, 492, 534, 518, 497, 502, 501, 526, 515, 534, 586, 511, 501, 543, 530, 499, 487, 521, 500, 499, 514, 491, 518, 534, 524, 479, 514, 493] payload_rows_per_dst=[5526, 5540, 5560, 5510, 5516, 5574, 5466, 5514] dup_rows_per_dst=[2746, 2600, 2602, 2632, 2666, 2770, 2694, 2620] combine_rows_per_src=[5526, 5540, 5560, 5510, 5516, 5574, 5466, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44648 payload_rows_remote=38708 dup_rows=20888 dup_rows_remote=17858 dedup_ratio=0.318726 router_tokens_per_expert=[510, 478, 517, 528, 501, 524, 520, 540, 483, 534, 520, 519, 510, 553, 493, 489, 521, 520, 535, 528, 480, 539, 509, 509, 513, 510, 529, 521, 521, 527, 513, 483, 537, 486, 509, 515, 516, 514, 511, 536, 485, 481, 496, 494, 490, 481, 541, 529, 478, 525, 494, 552, 473, 465, 532, 469, 517, 490, 517, 551, 512, 517, 536, 542] payload_rows_per_dst=[5562, 5348, 5554, 5420, 5652, 5544, 5628, 5940] dup_rows_per_dst=[2674, 2248, 2694, 2416, 2630, 2494, 2702, 3030] combine_rows_per_src=[5562, 5348, 5554, 5420, 5652, 5544, 5628, 5940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159375360 dispatch_meta_bytes=229392 combine_payload_bytes=159375360 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159604752 ingress_bytes=209707008 link_bytes=209707008 token_bytes=318980112 weight_movement_bytes=50331648 total_nvlink_bytes=369311760 weight_to_token_ratio=0.157789 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595628028e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=228840 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159579624 ingress_bytes=234848256 link_bytes=234848256 token_bytes=318930408 weight_movement_bytes=75497472 total_nvlink_bytes=394427880 weight_to_token_ratio=0.236721 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786923876e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158711808 dispatch_meta_bytes=229944 combine_payload_bytes=158711808 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158941752 ingress_bytes=234209280 link_bytes=234209280 token_bytes=317653560 weight_movement_bytes=75497472 total_nvlink_bytes=393151032 weight_to_token_ratio=0.237672 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782062007e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159318016 dispatch_meta_bytes=229280 combine_payload_bytes=159318016 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159547296 ingress_bytes=184483840 link_bytes=184483840 token_bytes=318865312 weight_movement_bytes=25165824 total_nvlink_bytes=344031136 weight_to_token_ratio=0.078923 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403708863e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158294016 dispatch_meta_bytes=229496 combine_payload_bytes=158294016 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158523512 ingress_bytes=208625664 link_bytes=208625664 token_bytes=316817528 weight_movement_bytes=50331648 total_nvlink_bytes=367149176 weight_to_token_ratio=0.158866 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.587400249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=38934 dup_rows=21072 dup_rows_remote=18540 dedup_ratio=0.321533 router_tokens_per_expert=[502, 511, 522, 522, 532, 506, 539, 480, 531, 555, 525, 528, 518, 520, 484, 459, 538, 531, 528, 555, 488, 494, 502, 474, 536, 533, 536, 529, 538, 534, 486, 528, 534, 508, 502, 459, 446, 530, 537, 506, 527, 492, 487, 488, 480, 532, 520, 505, 549, 476, 503, 486, 490, 526, 536, 500, 534, 513, 527, 524, 477, 521, 490, 499] payload_rows_per_dst=[5572, 5598, 5640, 5644, 5476, 5530, 5504, 5500] dup_rows_per_dst=[2656, 2642, 2580, 2796, 2568, 2532, 2628, 2670] combine_rows_per_src=[5572, 5598, 5640, 5644, 5476, 5530, 5504, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44426 payload_rows_remote=38902 dup_rows=21110 dup_rows_remote=18458 dedup_ratio=0.322113 router_tokens_per_expert=[492, 487, 473, 541, 514, 526, 493, 512, 485, 475, 518, 501, 540, 529, 491, 524, 498, 533, 525, 488, 520, 495, 512, 527, 496, 494, 500, 505, 499, 529, 544, 518, 531, 526, 499, 498, 526, 519, 553, 537, 499, 529, 549, 500, 516, 542, 528, 530, 546, 508, 504, 489, 485, 519, 540, 497, 504, 484, 488, 489, 496, 539, 519, 495] payload_rows_per_dst=[5542, 5500, 5544, 5584, 5666, 5572, 5524, 5494] dup_rows_per_dst=[2534, 2626, 2652, 2586, 2712, 2814, 2652, 2534] combine_rows_per_src=[5542, 5500, 5544, 5584, 5666, 5572, 5524, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158547968 dispatch_meta_bytes=226264 combine_payload_bytes=158547968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158774232 ingress_bytes=183713792 link_bytes=183713792 token_bytes=317322200 weight_movement_bytes=25165824 total_nvlink_bytes=342488024 weight_to_token_ratio=0.079307 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397849688e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159473664 dispatch_meta_bytes=229896 combine_payload_bytes=159473664 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159703560 ingress_bytes=184639488 link_bytes=184639488 token_bytes=319177224 weight_movement_bytes=25165824 total_nvlink_bytes=344343048 weight_to_token_ratio=0.078846 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404893165e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=229440 combine_payload_bytes=159342592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159572032 ingress_bytes=184508416 link_bytes=184508416 token_bytes=318914624 weight_movement_bytes=25165824 total_nvlink_bytes=344080448 weight_to_token_ratio=0.078911 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403895858e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39078 dup_rows=20934 dup_rows_remote=18362 dedup_ratio=0.319427 router_tokens_per_expert=[495, 503, 497, 477, 522, 525, 523, 506, 517, 521, 505, 549, 517, 498, 550, 478, 508, 504, 491, 483, 503, 521, 504, 487, 538, 506, 514, 536, 524, 495, 526, 542, 511, 493, 539, 522, 519, 473, 511, 546, 539, 522, 507, 497, 502, 451, 487, 518, 523, 514, 494, 523, 489, 521, 512, 505, 545, 534, 542, 466, 578, 490, 537, 493] payload_rows_per_dst=[5524, 5630, 5506, 5666, 5538, 5494, 5600, 5644] dup_rows_per_dst=[2572, 2700, 2514, 2678, 2630, 2552, 2562, 2726] combine_rows_per_src=[5524, 5630, 5506, 5666, 5538, 5494, 5600, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=39010 dup_rows=20982 dup_rows_remote=18248 dedup_ratio=0.320160 router_tokens_per_expert=[508, 548, 519, 505, 547, 526, 525, 542, 481, 530, 556, 495, 499, 518, 514, 531, 538, 441, 521, 505, 551, 512, 522, 490, 456, 533, 509, 524, 530, 548, 513, 526, 489, 494, 478, 510, 498, 513, 509, 549, 508, 503, 515, 489, 504, 533, 529, 509, 526, 483, 520, 473, 550, 493, 509, 513, 504, 534, 493, 468, 500, 515, 519, 475] payload_rows_per_dst=[5688, 5640, 5588, 5544, 5504, 5542, 5522, 5526] dup_rows_per_dst=[2752, 2608, 2572, 2734, 2576, 2638, 2612, 2490] combine_rows_per_src=[5688, 5640, 5588, 5544, 5504, 5542, 5522, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44526 payload_rows_remote=38974 dup_rows=21010 dup_rows_remote=18400 dedup_ratio=0.320587 router_tokens_per_expert=[516, 560, 512, 491, 512, 504, 464, 486, 518, 548, 531, 532, 525, 501, 545, 493, 533, 521, 517, 511, 508, 525, 486, 556, 529, 497, 526, 517, 497, 540, 521, 493, 492, 544, 509, 485, 475, 542, 511, 523, 497, 499, 544, 545, 523, 485, 521, 504, 498, 498, 506, 521, 487, 505, 498, 521, 527, 519, 526, 490, 508, 481, 489, 480] payload_rows_per_dst=[5554, 5640, 5632, 5608, 5552, 5548, 5482, 5510] dup_rows_per_dst=[2536, 2746, 2682, 2632, 2610, 2688, 2586, 2530] combine_rows_per_src=[5554, 5640, 5632, 5608, 5552, 5548, 5482, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=39000 dup_rows=20910 dup_rows_remote=18234 dedup_ratio=0.319061 router_tokens_per_expert=[497, 537, 472, 506, 516, 494, 499, 519, 498, 535, 506, 524, 480, 533, 531, 544, 496, 539, 465, 500, 500, 508, 532, 501, 496, 491, 486, 522, 506, 519, 532, 495, 532, 525, 515, 545, 499, 533, 564, 498, 524, 542, 493, 518, 512, 548, 473, 514, 523, 518, 476, 505, 527, 529, 555, 496, 503, 517, 502, 507, 475, 508, 514, 499] payload_rows_per_dst=[5492, 5626, 5592, 5536, 5622, 5602, 5624, 5532] dup_rows_per_dst=[2588, 2676, 2490, 2558, 2800, 2646, 2634, 2518] combine_rows_per_src=[5492, 5626, 5592, 5536, 5622, 5602, 5624, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38942 dup_rows=21076 dup_rows_remote=18522 dedup_ratio=0.321594 router_tokens_per_expert=[507, 494, 533, 527, 525, 484, 510, 513, 520, 478, 532, 492, 497, 527, 489, 538, 512, 505, 500, 502, 494, 487, 518, 518, 490, 527, 523, 524, 561, 565, 518, 517, 527, 488, 524, 475, 522, 543, 487, 501, 524, 525, 484, 522, 528, 542, 502, 533, 543, 541, 491, 519, 520, 470, 528, 484, 539, 478, 526, 498, 493, 499, 471, 514] payload_rows_per_dst=[5528, 5600, 5518, 5718, 5510, 5542, 5560, 5484] dup_rows_per_dst=[2658, 2546, 2554, 2732, 2624, 2778, 2632, 2552] combine_rows_per_src=[5528, 5600, 5518, 5718, 5510, 5542, 5560, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44410 payload_rows_remote=38716 dup_rows=21126 dup_rows_remote=18326 dedup_ratio=0.322357 router_tokens_per_expert=[519, 485, 510, 529, 506, 526, 508, 527, 484, 519, 527, 528, 530, 494, 537, 510, 508, 493, 542, 523, 508, 501, 522, 520, 484, 509, 483, 510, 510, 532, 494, 516, 510, 494, 563, 474, 516, 558, 501, 478, 494, 534, 499, 508, 537, 510, 491, 531, 530, 531, 480, 559, 483, 520, 501, 519, 524, 519, 501, 528, 481, 502, 507, 491] payload_rows_per_dst=[5466, 5346, 5410, 5494, 5764, 5594, 5642, 5694] dup_rows_per_dst=[2462, 2444, 2436, 2582, 2892, 2614, 2896, 2800] combine_rows_per_src=[5466, 5346, 5410, 5494, 5764, 5594, 5642, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=229760 combine_payload_bytes=160063488 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160293248 ingress_bytes=210395136 link_bytes=210395136 token_bytes=320356736 weight_movement_bytes=50331648 total_nvlink_bytes=370688384 weight_to_token_ratio=0.157111 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600863887e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=229032 combine_payload_bytes=159784960 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160013992 ingress_bytes=210116608 link_bytes=210116608 token_bytes=319798952 weight_movement_bytes=50331648 total_nvlink_bytes=370130600 weight_to_token_ratio=0.157385 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598744611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159744000 dispatch_meta_bytes=228936 combine_payload_bytes=159744000 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159972936 ingress_bytes=210075648 link_bytes=210075648 token_bytes=319716936 weight_movement_bytes=50331648 total_nvlink_bytes=370048584 weight_to_token_ratio=0.157426 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598432952e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159637504 dispatch_meta_bytes=229496 combine_payload_bytes=159637504 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159867000 ingress_bytes=209969152 link_bytes=209969152 token_bytes=319504504 weight_movement_bytes=50331648 total_nvlink_bytes=369836152 weight_to_token_ratio=0.157530 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597622641e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38722 dup_rows=21114 dup_rows_remote=18554 dedup_ratio=0.322174 router_tokens_per_expert=[506, 523, 495, 518, 500, 484, 485, 520, 525, 520, 510, 532, 542, 510, 488, 557, 504, 529, 526, 487, 510, 506, 477, 530, 508, 511, 517, 505, 520, 508, 516, 497, 535, 519, 518, 520, 484, 509, 507, 502, 514, 541, 486, 491, 509, 517, 544, 528, 513, 512, 513, 506, 541, 492, 510, 517, 512, 479, 519, 519, 515, 501, 488, 541] payload_rows_per_dst=[5448, 5604, 5542, 5572, 5524, 5700, 5534, 5498] dup_rows_per_dst=[2614, 2764, 2596, 2592, 2664, 2560, 2674, 2650] combine_rows_per_src=[5448, 5604, 5542, 5572, 5524, 5700, 5534, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=229856 combine_payload_bytes=159506432 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159736288 ingress_bytes=209838080 link_bytes=209838080 token_bytes=319242720 weight_movement_bytes=50331648 total_nvlink_bytes=369574368 weight_to_token_ratio=0.157660 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596625334e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44518 payload_rows_remote=38906 dup_rows=21018 dup_rows_remote=18436 dedup_ratio=0.320709 router_tokens_per_expert=[536, 479, 504, 539, 543, 504, 525, 513, 520, 519, 525, 525, 500, 505, 513, 502, 525, 528, 528, 544, 512, 539, 553, 495, 534, 468, 493, 522, 473, 501, 527, 541, 511, 513, 499, 500, 504, 523, 488, 518, 513, 490, 527, 487, 533, 511, 501, 515, 532, 525, 526, 531, 476, 504, 513, 490, 471, 509, 456, 502, 514, 466, 554, 531] payload_rows_per_dst=[5616, 5628, 5640, 5480, 5516, 5570, 5612, 5456] dup_rows_per_dst=[2670, 2590, 2808, 2638, 2596, 2584, 2582, 2550] combine_rows_per_src=[5616, 5628, 5640, 5480, 5516, 5570, 5612, 5456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158580736 dispatch_meta_bytes=228168 combine_payload_bytes=158580736 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158808904 ingress_bytes=183746560 link_bytes=183746560 token_bytes=317389640 weight_movement_bytes=25165824 total_nvlink_bytes=342555464 weight_to_token_ratio=0.079290 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398099014e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158605312 dispatch_meta_bytes=229104 combine_payload_bytes=158605312 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=158834416 ingress_bytes=259268608 link_bytes=259268608 token_bytes=317439728 weight_movement_bytes=100663296 total_nvlink_bytes=418103024 weight_to_token_ratio=0.317110 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.972734539e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229368 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159588344 ingress_bytes=184524800 link_bytes=184524800 token_bytes=318947320 weight_movement_bytes=25165824 total_nvlink_bytes=344113144 weight_to_token_ratio=0.078903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404020521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44682 payload_rows_remote=39130 dup_rows=20854 dup_rows_remote=18206 dedup_ratio=0.318207 router_tokens_per_expert=[500, 519, 515, 506, 509, 477, 534, 514, 572, 489, 515, 510, 526, 530, 502, 519, 468, 502, 523, 524, 511, 560, 541, 519, 533, 477, 499, 506, 467, 503, 495, 534, 514, 468, 520, 547, 531, 489, 517, 514, 484, 530, 484, 514, 478, 525, 510, 508, 512, 522, 526, 490, 481, 525, 549, 518, 529, 501, 507, 554, 508, 516, 486, 512] payload_rows_per_dst=[5552, 5794, 5598, 5536, 5432, 5532, 5636, 5602] dup_rows_per_dst=[2648, 2698, 2646, 2492, 2602, 2534, 2610, 2624] combine_rows_per_src=[5552, 5794, 5598, 5536, 5432, 5532, 5636, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44330 payload_rows_remote=38808 dup_rows=21206 dup_rows_remote=18548 dedup_ratio=0.323578 router_tokens_per_expert=[513, 520, 517, 516, 547, 538, 506, 502, 540, 514, 556, 530, 506, 505, 523, 469, 490, 507, 495, 497, 531, 505, 489, 521, 501, 481, 544, 517, 506, 522, 516, 503, 516, 551, 510, 478, 490, 486, 501, 514, 498, 507, 542, 509, 516, 536, 504, 516, 533, 525, 499, 512, 491, 465, 496, 500, 504, 502, 523, 525, 542, 497, 536, 517] payload_rows_per_dst=[5534, 5556, 5476, 5522, 5490, 5620, 5498, 5634] dup_rows_per_dst=[2784, 2730, 2594, 2658, 2602, 2636, 2544, 2658] combine_rows_per_src=[5534, 5556, 5476, 5522, 5490, 5620, 5498, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44520 payload_rows_remote=38998 dup_rows=21016 dup_rows_remote=18324 dedup_ratio=0.320679 router_tokens_per_expert=[485, 501, 514, 526, 524, 502, 502, 513, 522, 534, 492, 513, 535, 514, 519, 478, 512, 518, 529, 502, 541, 503, 517, 518, 512, 563, 469, 525, 515, 535, 492, 493, 503, 522, 476, 541, 519, 552, 489, 483, 503, 527, 510, 493, 514, 526, 483, 522, 533, 491, 483, 556, 510, 550, 522, 514, 477, 508, 487, 506, 502, 528, 487, 533] payload_rows_per_dst=[5568, 5522, 5582, 5588, 5594, 5598, 5588, 5480] dup_rows_per_dst=[2566, 2692, 2698, 2620, 2576, 2558, 2730, 2576] combine_rows_per_src=[5568, 5522, 5582, 5588, 5594, 5598, 5588, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38848 dup_rows=21116 dup_rows_remote=18562 dedup_ratio=0.322205 router_tokens_per_expert=[508, 505, 539, 497, 531, 526, 507, 445, 516, 541, 545, 526, 479, 494, 511, 538, 491, 515, 546, 517, 485, 535, 504, 519, 520, 531, 531, 531, 521, 513, 517, 510, 519, 523, 490, 500, 485, 526, 518, 502, 530, 513, 492, 519, 527, 489, 522, 493, 512, 518, 485, 484, 480, 461, 549, 482, 491, 513, 523, 542, 532, 503, 530, 521] payload_rows_per_dst=[5506, 5558, 5568, 5654, 5572, 5570, 5394, 5598] dup_rows_per_dst=[2610, 2742, 2656, 2694, 2554, 2600, 2548, 2712] combine_rows_per_src=[5506, 5558, 5568, 5654, 5572, 5570, 5394, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44608 payload_rows_remote=38964 dup_rows=20928 dup_rows_remote=18428 dedup_ratio=0.319336 router_tokens_per_expert=[524, 527, 514, 507, 570, 504, 514, 514, 524, 496, 500, 531, 520, 500, 471, 496, 501, 490, 517, 478, 526, 528, 528, 504, 511, 487, 489, 548, 521, 535, 497, 530, 531, 528, 538, 497, 512, 490, 507, 458, 507, 526, 514, 501, 462, 478, 548, 526, 525, 483, 494, 535, 528, 522, 537, 493, 514, 501, 536, 531, 489, 504, 525, 526] payload_rows_per_dst=[5614, 5536, 5644, 5590, 5550, 5546, 5522, 5606] dup_rows_per_dst=[2734, 2540, 2500, 2646, 2572, 2578, 2712, 2646] combine_rows_per_src=[5614, 5536, 5644, 5590, 5550, 5546, 5522, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44254 payload_rows_remote=38646 dup_rows=21282 dup_rows_remote=18512 dedup_ratio=0.324738 router_tokens_per_expert=[532, 519, 502, 481, 503, 502, 517, 478, 537, 511, 518, 531, 507, 525, 507, 494, 524, 471, 544, 557, 521, 541, 499, 515, 511, 470, 490, 503, 516, 542, 559, 507, 497, 505, 555, 507, 516, 490, 551, 502, 476, 532, 491, 541, 512, 502, 539, 548, 498, 518, 525, 497, 505, 501, 490, 488, 483, 491, 521, 509, 511, 512, 504, 517] payload_rows_per_dst=[5538, 5396, 5576, 5392, 5724, 5558, 5462, 5608] dup_rows_per_dst=[2530, 2396, 2768, 2522, 2990, 2704, 2602, 2770] combine_rows_per_src=[5538, 5396, 5576, 5392, 5724, 5558, 5462, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=160276480 dispatch_meta_bytes=229344 combine_payload_bytes=160276480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160505824 ingress_bytes=185442304 link_bytes=185442304 token_bytes=320782304 weight_movement_bytes=25165824 total_nvlink_bytes=345948128 weight_to_token_ratio=0.078451 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411001667e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38792 dup_rows=21076 dup_rows_remote=18362 dedup_ratio=0.321594 router_tokens_per_expert=[485, 491, 487, 509, 509, 470, 520, 516, 532, 487, 542, 507, 546, 500, 553, 502, 479, 527, 501, 471, 493, 521, 495, 509, 516, 538, 556, 499, 512, 487, 473, 508, 481, 485, 493, 504, 516, 476, 546, 477, 538, 508, 537, 506, 500, 525, 560, 517, 492, 519, 551, 505, 521, 518, 546, 527, 485, 514, 521, 506, 504, 541, 553, 555] payload_rows_per_dst=[5478, 5586, 5496, 5510, 5444, 5668, 5630, 5648] dup_rows_per_dst=[2496, 2752, 2496, 2668, 2512, 2714, 2728, 2710] combine_rows_per_src=[5478, 5586, 5496, 5510, 5444, 5668, 5630, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159121408 dispatch_meta_bytes=229640 combine_payload_bytes=159121408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159351048 ingress_bytes=184287232 link_bytes=184287232 token_bytes=318472456 weight_movement_bytes=25165824 total_nvlink_bytes=343638280 weight_to_token_ratio=0.079020 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402212903e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158957568 dispatch_meta_bytes=229424 combine_payload_bytes=158957568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159186992 ingress_bytes=184123392 link_bytes=184123392 token_bytes=318144560 weight_movement_bytes=25165824 total_nvlink_bytes=343310384 weight_to_token_ratio=0.079102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400966270e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159735808 dispatch_meta_bytes=229288 combine_payload_bytes=159735808 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159965096 ingress_bytes=210067456 link_bytes=210067456 token_bytes=319700904 weight_movement_bytes=50331648 total_nvlink_bytes=370032552 weight_to_token_ratio=0.157434 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598370621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44744 payload_rows_remote=39134 dup_rows=20792 dup_rows_remote=18182 dedup_ratio=0.317261 router_tokens_per_expert=[519, 511, 566, 481, 524, 458, 492, 488, 519, 545, 481, 531, 508, 521, 497, 525, 530, 536, 509, 505, 483, 527, 533, 516, 527, 522, 495, 504, 538, 526, 503, 484, 527, 492, 475, 513, 455, 531, 517, 494, 514, 510, 502, 545, 499, 467, 500, 550, 509, 493, 488, 497, 534, 523, 539, 527, 550, 500, 515, 545, 504, 502, 512, 535] payload_rows_per_dst=[5540, 5602, 5596, 5626, 5510, 5536, 5610, 5724] dup_rows_per_dst=[2538, 2652, 2682, 2572, 2498, 2638, 2610, 2602] combine_rows_per_src=[5540, 5602, 5596, 5626, 5510, 5536, 5610, 5724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229568 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159826112 ingress_bytes=209928192 link_bytes=209928192 token_bytes=319422656 weight_movement_bytes=50331648 total_nvlink_bytes=369754304 weight_to_token_ratio=0.157571 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597310982e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158294016 dispatch_meta_bytes=228632 combine_payload_bytes=158294016 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158522648 ingress_bytes=208625664 link_bytes=208625664 token_bytes=316816664 weight_movement_bytes=50331648 total_nvlink_bytes=367148312 weight_to_token_ratio=0.158867 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.587400249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158892032 dispatch_meta_bytes=228616 combine_payload_bytes=158892032 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159120648 ingress_bytes=209223680 link_bytes=209223680 token_bytes=318012680 weight_movement_bytes=50331648 total_nvlink_bytes=368344328 weight_to_token_ratio=0.158269 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591950460e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160292864 dispatch_meta_bytes=229264 combine_payload_bytes=160292864 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160522128 ingress_bytes=210624512 link_bytes=210624512 token_bytes=320814992 weight_movement_bytes=50331648 total_nvlink_bytes=371146640 weight_to_token_ratio=0.156887 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602609173e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5086823584 prefetch_bytes=327155712 reduce_bytes=276824064 weight_movement_bytes=603979776 egress_bytes=2545239200 ingress_bytes=3145564160 link_bytes=3145564160 total_nvlink_bytes=5690803360 weight_to_token_ratio=0.118734 prefetch_share_of_ingress=0.104005 dedup_ratio=0.321846 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=11 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.495881797e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44350 payload_rows_remote=38676 dup_rows=21186 dup_rows_remote=18516 dedup_ratio=0.323273 router_tokens_per_expert=[492, 548, 547, 519, 535, 512, 553, 461, 471, 533, 561, 528, 501, 540, 496, 504, 558, 548, 500, 490, 516, 522, 491, 548, 506, 512, 493, 491, 505, 564, 465, 553, 472, 488, 505, 495, 493, 513, 500, 569, 513, 530, 517, 516, 499, 510, 534, 532, 500, 532, 476, 490, 520, 490, 524, 498, 521, 501, 514, 487, 469, 520, 469, 508] payload_rows_per_dst=[5674, 5568, 5564, 5524, 5504, 5550, 5502, 5464] dup_rows_per_dst=[2670, 2716, 2822, 2682, 2566, 2658, 2558, 2514] combine_rows_per_src=[5674, 5568, 5564, 5524, 5504, 5550, 5502, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5107349440 prefetch_bytes=327155712 reduce_bytes=427819008 weight_movement_bytes=754974720 egress_bytes=2555508672 ingress_bytes=3306815488 link_bytes=3306815488 total_nvlink_bytes=5862324160 weight_to_token_ratio=0.147821 prefetch_share_of_ingress=0.098934 dedup_ratio=0.320623 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=17 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.572565316e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44592 payload_rows_remote=39078 dup_rows=20944 dup_rows_remote=18330 dedup_ratio=0.319580 router_tokens_per_expert=[528, 472, 520, 508, 510, 493, 505, 501, 530, 503, 512, 503, 528, 504, 524, 561, 503, 507, 527, 531, 499, 487, 517, 529, 509, 526, 502, 503, 516, 507, 497, 522, 520, 504, 507, 522, 521, 501, 499, 490, 538, 508, 526, 494, 483, 508, 457, 536, 532, 511, 537, 549, 497, 509, 449, 496, 515, 529, 510, 538, 507, 511, 538, 542] payload_rows_per_dst=[5534, 5560, 5578, 5556, 5514, 5582, 5624, 5644] dup_rows_per_dst=[2540, 2770, 2622, 2608, 2614, 2518, 2536, 2736] combine_rows_per_src=[5534, 5560, 5578, 5556, 5514, 5582, 5624, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44604 payload_rows_remote=39030 dup_rows=20932 dup_rows_remote=18280 dedup_ratio=0.319397 router_tokens_per_expert=[505, 487, 502, 512, 520, 530, 488, 543, 463, 491, 522, 503, 521, 529, 529, 514, 521, 491, 471, 545, 529, 519, 500, 543, 532, 523, 475, 537, 525, 499, 469, 553, 518, 507, 534, 497, 525, 514, 545, 503, 505, 522, 493, 500, 521, 501, 496, 476, 535, 542, 534, 518, 505, 522, 498, 523, 492, 525, 519, 491, 491, 524, 499, 502] payload_rows_per_dst=[5586, 5540, 5616, 5574, 5594, 5556, 5644, 5494] dup_rows_per_dst=[2588, 2604, 2622, 2652, 2692, 2472, 2710, 2592] combine_rows_per_src=[5586, 5540, 5616, 5574, 5594, 5556, 5644, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44482 payload_rows_remote=38852 dup_rows=21054 dup_rows_remote=18386 dedup_ratio=0.321259 router_tokens_per_expert=[485, 535, 538, 502, 479, 505, 522, 562, 536, 509, 504, 523, 520, 515, 538, 504, 481, 506, 543, 516, 495, 508, 501, 465, 516, 502, 494, 544, 507, 519, 484, 557, 521, 493, 497, 514, 488, 508, 531, 501, 537, 513, 497, 519, 510, 536, 536, 520, 509, 518, 544, 446, 500, 519, 509, 521, 517, 519, 495, 520, 511, 498, 508, 498] payload_rows_per_dst=[5534, 5630, 5466, 5634, 5512, 5564, 5556, 5586] dup_rows_per_dst=[2722, 2668, 2564, 2612, 2594, 2772, 2576, 2546] combine_rows_per_src=[5534, 5630, 5466, 5634, 5512, 5564, 5556, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5112727232 prefetch_bytes=327155712 reduce_bytes=226492416 weight_movement_bytes=553648128 egress_bytes=2558199488 ingress_bytes=3108175872 link_bytes=3108175872 total_nvlink_bytes=5666375360 weight_to_token_ratio=0.108288 prefetch_share_of_ingress=0.105256 dedup_ratio=0.320599 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=9 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.478101693e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38774 dup_rows=21102 dup_rows_remote=18422 dedup_ratio=0.321991 router_tokens_per_expert=[513, 470, 494, 506, 491, 506, 524, 523, 528, 518, 530, 522, 533, 500, 515, 470, 502, 540, 507, 540, 536, 523, 520, 502, 482, 502, 544, 510, 506, 540, 547, 486, 515, 508, 494, 524, 489, 516, 543, 500, 552, 523, 488, 499, 505, 483, 536, 536, 493, 496, 484, 510, 487, 520, 495, 512, 494, 504, 507, 522, 522, 518, 517, 546] payload_rows_per_dst=[5528, 5536, 5660, 5494, 5542, 5574, 5478, 5622] dup_rows_per_dst=[2526, 2696, 2680, 2740, 2636, 2670, 2516, 2638] combine_rows_per_src=[5528, 5536, 5660, 5494, 5542, 5574, 5478, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158416896 dispatch_meta_bytes=228768 combine_payload_bytes=158416896 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158645664 ingress_bytes=208748544 link_bytes=208748544 token_bytes=317062560 weight_movement_bytes=50331648 total_nvlink_bytes=367394208 weight_to_token_ratio=0.158744 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588335224e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=229632 combine_payload_bytes=160063488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160293120 ingress_bytes=185229312 link_bytes=185229312 token_bytes=320356608 weight_movement_bytes=25165824 total_nvlink_bytes=345522432 weight_to_token_ratio=0.078556 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409381044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=229240 combine_payload_bytes=159866880 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160096120 ingress_bytes=210198528 link_bytes=210198528 token_bytes=319963000 weight_movement_bytes=50331648 total_nvlink_bytes=370294648 weight_to_token_ratio=0.157305 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599367927e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159137792 dispatch_meta_bytes=228952 combine_payload_bytes=159137792 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159366744 ingress_bytes=209469440 link_bytes=209469440 token_bytes=318504536 weight_movement_bytes=50331648 total_nvlink_bytes=368836184 weight_to_token_ratio=0.158025 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593820410e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158818304 dispatch_meta_bytes=228784 combine_payload_bytes=158818304 prefetch_bytes=0 reduce_bytes=0 egress_bytes=159047088 ingress_bytes=158818304 link_bytes=159047088 token_bytes=317865392 weight_movement_bytes=0 total_nvlink_bytes=317865392 weight_to_token_ratio=0.000000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.210164571e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5105058304 prefetch_bytes=352321536 reduce_bytes=352321536 weight_movement_bytes=704643072 egress_bytes=2554364416 ingress_bytes=3255336960 link_bytes=3255336960 total_nvlink_bytes=5809701376 weight_to_token_ratio=0.138028 prefetch_share_of_ingress=0.108229 dedup_ratio=0.321117 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=14 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.548084558e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5107155424 prefetch_bytes=377487360 reduce_bytes=251658240 weight_movement_bytes=629145600 egress_bytes=2555412960 ingress_bytes=3180888064 link_bytes=3180888064 total_nvlink_bytes=5736301024 weight_to_token_ratio=0.123189 prefetch_share_of_ingress=0.118674 dedup_ratio=0.321106 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=10 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.512680178e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5104599568 prefetch_bytes=352321536 reduce_bytes=452984832 weight_movement_bytes=805306368 egress_bytes=2554135056 ingress_bytes=3355770880 link_bytes=3355770880 total_nvlink_bytes=5909905936 weight_to_token_ratio=0.157761 prefetch_share_of_ingress=0.104990 dedup_ratio=0.321369 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=18 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.595846189e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5101632016 prefetch_bytes=352321536 reduce_bytes=352321536 weight_movement_bytes=704643072 egress_bytes=2552650256 ingress_bytes=3253624832 link_bytes=3253853616 total_nvlink_bytes=5806275088 weight_to_token_ratio=0.138121 prefetch_share_of_ingress=0.108286 dedup_ratio=0.321362 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=14 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.547379150e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38546 dup_rows=21124 dup_rows_remote=18132 dedup_ratio=0.322327 router_tokens_per_expert=[507, 553, 505, 528, 485, 508, 502, 514, 522, 472, 484, 484, 508, 473, 516, 558, 479, 533, 482, 498, 514, 527, 537, 487, 505, 510, 511, 506, 544, 506, 519, 549, 504, 522, 496, 515, 538, 516, 489, 500, 491, 517, 522, 526, 489, 507, 497, 513, 491, 510, 522, 517, 499, 503, 572, 551, 504, 527, 525, 526, 487, 500, 516, 550] payload_rows_per_dst=[5866, 5496, 5542, 5612, 5512, 5478, 5566, 5340] dup_rows_per_dst=[2992, 2652, 2572, 2764, 2648, 2532, 2688, 2276] combine_rows_per_src=[5866, 5496, 5542, 5612, 5512, 5478, 5566, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=157884416 dispatch_meta_bytes=226712 combine_payload_bytes=157884416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158111128 ingress_bytes=183050240 link_bytes=183050240 token_bytes=315995544 weight_movement_bytes=25165824 total_nvlink_bytes=341161368 weight_to_token_ratio=0.079640 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.392800824e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5092694704 prefetch_bytes=327155712 reduce_bytes=402653184 weight_movement_bytes=729808896 egress_bytes=2548177584 ingress_bytes=3274326016 link_bytes=3274326016 total_nvlink_bytes=5822503600 weight_to_token_ratio=0.143305 prefetch_share_of_ingress=0.099915 dedup_ratio=0.321594 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=16 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.557114857e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none + [2026-09-21 08:32:50] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 210.5 | throughput per GPU (TFLOP/s/GPU): 153.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.773955E+00 | loss scale: 1.0 | grad norm: 3.820 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44538 payload_rows_remote=39022 dup_rows=20998 dup_rows_remote=18298 dedup_ratio=0.320404 router_tokens_per_expert=[524, 508, 533, 518, 521, 496, 509, 499, 504, 533, 523, 518, 482, 508, 498, 556, 494, 500, 494, 528, 532, 518, 520, 522, 530, 485, 485, 525, 529, 481, 554, 540, 504, 552, 530, 535, 502, 486, 496, 511, 519, 496, 465, 488, 476, 502, 527, 496, 518, 508, 537, 525, 543, 534, 505, 559, 509, 506, 505, 493, 485, 504, 505, 480] payload_rows_per_dst=[5550, 5532, 5516, 5600, 5612, 5540, 5728, 5460] dup_rows_per_dst=[2666, 2712, 2700, 2658, 2620, 2398, 2730, 2514] combine_rows_per_src=[5550, 5532, 5516, 5600, 5612, 5540, 5728, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39064 dup_rows=20934 dup_rows_remote=18340 dedup_ratio=0.319427 router_tokens_per_expert=[524, 542, 546, 505, 516, 511, 498, 517, 504, 516, 485, 507, 537, 510, 502, 500, 463, 532, 519, 458, 484, 479, 512, 505, 504, 496, 529, 507, 508, 501, 491, 512, 554, 535, 518, 529, 529, 505, 484, 510, 480, 530, 518, 542, 478, 496, 530, 564, 491, 536, 500, 500, 498, 501, 523, 517, 511, 559, 504, 497, 505, 550, 498, 556] payload_rows_per_dst=[5644, 5514, 5466, 5582, 5596, 5604, 5538, 5658] dup_rows_per_dst=[2674, 2608, 2438, 2514, 2732, 2672, 2594, 2702] combine_rows_per_src=[5644, 5514, 5466, 5582, 5596, 5604, 5538, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44456 payload_rows_remote=38904 dup_rows=21080 dup_rows_remote=18508 dedup_ratio=0.321655 router_tokens_per_expert=[502, 512, 523, 557, 460, 481, 478, 495, 511, 536, 498, 472, 504, 566, 484, 501, 477, 549, 522, 539, 483, 502, 504, 473, 509, 540, 488, 520, 517, 486, 555, 492, 507, 520, 468, 511, 527, 528, 524, 477, 515, 515, 555, 553, 552, 520, 543, 527, 515, 496, 516, 521, 511, 517, 509, 485, 491, 527, 485, 520, 550, 539, 531, 477] payload_rows_per_dst=[5470, 5560, 5520, 5590, 5552, 5684, 5494, 5586] dup_rows_per_dst=[2546, 2584, 2578, 2624, 2572, 2876, 2646, 2654] combine_rows_per_src=[5470, 5560, 5520, 5590, 5552, 5684, 5494, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44432 payload_rows_remote=38896 dup_rows=21104 dup_rows_remote=18504 dedup_ratio=0.322021 router_tokens_per_expert=[507, 519, 495, 500, 518, 492, 547, 477, 577, 520, 524, 498, 476, 487, 533, 542, 509, 534, 480, 510, 497, 532, 550, 515, 547, 522, 496, 503, 485, 515, 531, 469, 533, 522, 537, 513, 509, 483, 513, 443, 539, 528, 509, 521, 471, 543, 505, 558, 494, 505, 543, 486, 497, 514, 498, 489, 497, 541, 522, 516, 531, 509, 512, 480] payload_rows_per_dst=[5488, 5584, 5574, 5536, 5518, 5654, 5480, 5598] dup_rows_per_dst=[2622, 2730, 2680, 2600, 2588, 2694, 2572, 2618] combine_rows_per_src=[5488, 5584, 5574, 5536, 5518, 5654, 5480, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44322 payload_rows_remote=38752 dup_rows=21214 dup_rows_remote=18540 dedup_ratio=0.323700 router_tokens_per_expert=[501, 496, 529, 540, 537, 518, 501, 506, 494, 519, 514, 515, 490, 536, 486, 490, 498, 541, 527, 509, 530, 520, 470, 459, 513, 513, 520, 527, 529, 498, 504, 487, 546, 504, 534, 489, 510, 501, 550, 547, 517, 525, 508, 482, 480, 537, 564, 509, 503, 497, 547, 521, 501, 522, 457, 557, 504, 496, 477, 524, 526, 524, 495, 497] payload_rows_per_dst=[5550, 5528, 5500, 5496, 5622, 5570, 5618, 5438] dup_rows_per_dst=[2706, 2560, 2608, 2686, 2740, 2674, 2592, 2648] combine_rows_per_src=[5550, 5528, 5500, 5496, 5622, 5570, 5618, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=39024 dup_rows=20988 dup_rows_remote=18450 dedup_ratio=0.320251 router_tokens_per_expert=[502, 476, 546, 525, 546, 517, 494, 468, 532, 501, 453, 512, 534, 542, 487, 518, 525, 482, 508, 541, 539, 548, 512, 511, 548, 500, 523, 472, 496, 517, 546, 505, 535, 548, 509, 492, 514, 522, 509, 477, 512, 517, 514, 527, 487, 531, 521, 519, 539, 475, 526, 533, 509, 508, 469, 527, 467, 549, 489, 515, 493, 508, 524, 477] payload_rows_per_dst=[5500, 5542, 5648, 5622, 5580, 5590, 5542, 5524] dup_rows_per_dst=[2570, 2616, 2684, 2670, 2632, 2658, 2620, 2538] combine_rows_per_src=[5500, 5542, 5648, 5622, 5580, 5590, 5542, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44582 payload_rows_remote=39046 dup_rows=20954 dup_rows_remote=18444 dedup_ratio=0.319733 router_tokens_per_expert=[505, 524, 511, 478, 544, 504, 536, 529, 512, 496, 479, 520, 535, 521, 491, 469, 483, 560, 488, 523, 493, 496, 501, 511, 515, 508, 534, 529, 500, 502, 505, 552, 552, 491, 519, 483, 510, 516, 492, 485, 496, 486, 486, 520, 516, 569, 528, 505, 502, 508, 533, 506, 500, 521, 536, 526, 543, 479, 517, 530, 538, 489, 518, 514] payload_rows_per_dst=[5578, 5536, 5546, 5572, 5534, 5610, 5574, 5632] dup_rows_per_dst=[2684, 2510, 2564, 2718, 2562, 2602, 2690, 2624] combine_rows_per_src=[5578, 5536, 5546, 5572, 5534, 5610, 5574, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159834112 dispatch_meta_bytes=229280 combine_payload_bytes=159834112 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160063392 ingress_bytes=184999936 link_bytes=184999936 token_bytes=319897504 weight_movement_bytes=25165824 total_nvlink_bytes=345063328 weight_to_token_ratio=0.078668 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407635757e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160006144 dispatch_meta_bytes=229616 combine_payload_bytes=160006144 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160235760 ingress_bytes=235503616 link_bytes=235503616 token_bytes=320241904 weight_movement_bytes=75497472 total_nvlink_bytes=395739376 weight_to_token_ratio=0.235751 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791910408e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229648 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159580432 ingress_bytes=209682432 link_bytes=209682432 token_bytes=318931216 weight_movement_bytes=50331648 total_nvlink_bytes=369262864 weight_to_token_ratio=0.157813 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595441033e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159318016 dispatch_meta_bytes=229600 combine_payload_bytes=159318016 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159547616 ingress_bytes=184483840 link_bytes=184483840 token_bytes=318865632 weight_movement_bytes=25165824 total_nvlink_bytes=344031456 weight_to_token_ratio=0.078923 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403708863e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158728192 dispatch_meta_bytes=229168 combine_payload_bytes=158728192 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=158957360 ingress_bytes=234225664 link_bytes=234225664 token_bytes=317685552 weight_movement_bytes=75497472 total_nvlink_bytes=393183024 weight_to_token_ratio=0.237648 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782186670e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38782 dup_rows=21142 dup_rows_remote=18466 dedup_ratio=0.322601 router_tokens_per_expert=[517, 504, 514, 507, 541, 508, 518, 518, 541, 484, 479, 504, 492, 506, 523, 516, 498, 518, 522, 529, 538, 491, 532, 495, 537, 492, 515, 514, 483, 471, 492, 530, 517, 512, 541, 505, 520, 524, 555, 520, 510, 497, 519, 501, 515, 478, 483, 503, 487, 528, 531, 500, 470, 524, 509, 519, 524, 509, 491, 510, 527, 543, 534, 533] payload_rows_per_dst=[5612, 5576, 5632, 5566, 5530, 5368, 5474, 5636] dup_rows_per_dst=[2676, 2844, 2882, 2502, 2546, 2342, 2644, 2706] combine_rows_per_src=[5612, 5576, 5632, 5566, 5530, 5368, 5474, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159842304 dispatch_meta_bytes=229896 combine_payload_bytes=159842304 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160072200 ingress_bytes=185008128 link_bytes=185008128 token_bytes=319914504 weight_movement_bytes=25165824 total_nvlink_bytes=345080328 weight_to_token_ratio=0.078664 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407698089e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159932416 dispatch_meta_bytes=229960 combine_payload_bytes=159932416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160162376 ingress_bytes=185098240 link_bytes=185098240 token_bytes=320094792 weight_movement_bytes=25165824 total_nvlink_bytes=345260616 weight_to_token_ratio=0.078620 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408383737e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44498 payload_rows_remote=38950 dup_rows=21038 dup_rows_remote=18378 dedup_ratio=0.321014 router_tokens_per_expert=[497, 523, 561, 503, 541, 497, 562, 499, 510, 508, 511, 476, 527, 492, 562, 535, 515, 497, 510, 530, 512, 514, 501, 525, 492, 528, 518, 513, 521, 521, 493, 494, 521, 530, 534, 507, 527, 472, 493, 511, 507, 501, 517, 551, 516, 495, 508, 486, 479, 503, 530, 459, 527, 546, 522, 504, 497, 532, 466, 479, 524, 515, 509, 512] payload_rows_per_dst=[5630, 5572, 5548, 5502, 5652, 5514, 5582, 5498] dup_rows_per_dst=[2736, 2670, 2660, 2658, 2538, 2648, 2558, 2570] combine_rows_per_src=[5630, 5572, 5548, 5502, 5652, 5514, 5582, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38786 dup_rows=21124 dup_rows_remote=18450 dedup_ratio=0.322327 router_tokens_per_expert=[510, 535, 521, 520, 522, 494, 487, 461, 508, 496, 504, 529, 502, 529, 533, 531, 496, 516, 533, 507, 494, 532, 499, 512, 501, 485, 524, 514, 489, 533, 501, 549, 504, 557, 502, 470, 509, 525, 509, 574, 542, 505, 513, 512, 475, 507, 491, 490, 521, 521, 540, 524, 507, 505, 493, 511, 521, 511, 508, 496, 538, 501, 535, 484] payload_rows_per_dst=[5500, 5632, 5504, 5528, 5626, 5512, 5558, 5552] dup_rows_per_dst=[2600, 2632, 2674, 2664, 2674, 2558, 2686, 2636] combine_rows_per_src=[5500, 5632, 5504, 5528, 5626, 5512, 5558, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44366 payload_rows_remote=38840 dup_rows=21170 dup_rows_remote=18428 dedup_ratio=0.323029 router_tokens_per_expert=[507, 493, 522, 496, 512, 500, 499, 526, 541, 487, 523, 493, 513, 532, 473, 525, 510, 511, 493, 499, 507, 546, 513, 524, 482, 500, 555, 530, 501, 486, 514, 526, 504, 522, 511, 535, 515, 531, 511, 543, 538, 487, 474, 526, 505, 508, 517, 471, 533, 517, 489, 504, 570, 467, 558, 496, 531, 509, 508, 533, 516, 518, 487, 495] payload_rows_per_dst=[5528, 5546, 5566, 5566, 5586, 5490, 5526, 5558] dup_rows_per_dst=[2582, 2628, 2640, 2622, 2758, 2562, 2742, 2636] combine_rows_per_src=[5528, 5546, 5566, 5566, 5586, 5490, 5526, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158851072 dispatch_meta_bytes=228992 combine_payload_bytes=158851072 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159080064 ingress_bytes=209182720 link_bytes=209182720 token_bytes=317931136 weight_movement_bytes=50331648 total_nvlink_bytes=368262784 weight_to_token_ratio=0.158310 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591638802e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38984 dup_rows=21002 dup_rows_remote=18408 dedup_ratio=0.320465 router_tokens_per_expert=[535, 520, 496, 557, 513, 553, 543, 504, 482, 522, 511, 547, 534, 492, 501, 524, 534, 478, 512, 516, 494, 477, 523, 489, 472, 493, 469, 540, 504, 535, 521, 538, 462, 487, 549, 542, 536, 510, 501, 514, 457, 509, 500, 524, 528, 556, 475, 520, 524, 523, 547, 522, 501, 507, 525, 510, 498, 530, 483, 489, 468, 526, 488, 528] payload_rows_per_dst=[5612, 5590, 5478, 5550, 5598, 5526, 5636, 5544] dup_rows_per_dst=[2830, 2636, 2568, 2594, 2604, 2612, 2682, 2476] combine_rows_per_src=[5612, 5590, 5478, 5550, 5598, 5526, 5636, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44726 payload_rows_remote=39138 dup_rows=20810 dup_rows_remote=18144 dedup_ratio=0.317535 router_tokens_per_expert=[544, 516, 529, 559, 462, 492, 528, 501, 488, 507, 530, 498, 500, 512, 523, 542, 503, 507, 521, 533, 490, 535, 548, 538, 522, 461, 490, 538, 495, 501, 502, 556, 504, 482, 474, 503, 520, 480, 526, 509, 520, 563, 513, 480, 513, 500, 544, 494, 523, 486, 521, 524, 489, 497, 535, 519, 513, 524, 482, 507, 493, 538, 491, 530] payload_rows_per_dst=[5624, 5562, 5692, 5596, 5502, 5588, 5568, 5594] dup_rows_per_dst=[2638, 2638, 2658, 2534, 2494, 2666, 2620, 2562] combine_rows_per_src=[5624, 5562, 5692, 5596, 5502, 5588, 5568, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44532 payload_rows_remote=38796 dup_rows=21004 dup_rows_remote=18034 dedup_ratio=0.320496 router_tokens_per_expert=[512, 493, 538, 504, 533, 499, 478, 522, 522, 480, 475, 499, 527, 482, 503, 515, 523, 486, 507, 502, 539, 513, 505, 489, 492, 504, 512, 524, 523, 510, 527, 480, 505, 526, 497, 518, 531, 529, 510, 521, 516, 520, 500, 486, 543, 535, 499, 500, 504, 505, 489, 512, 525, 539, 505, 537, 528, 520, 504, 529, 527, 542, 524, 524] payload_rows_per_dst=[5388, 5508, 5460, 5622, 5624, 5646, 5548, 5736] dup_rows_per_dst=[2400, 2498, 2440, 2666, 2748, 2552, 2730, 2970] combine_rows_per_src=[5388, 5508, 5460, 5622, 5624, 5646, 5548, 5736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38840 dup_rows=21102 dup_rows_remote=18458 dedup_ratio=0.321991 router_tokens_per_expert=[545, 480, 466, 527, 536, 539, 483, 514, 552, 495, 483, 524, 482, 548, 502, 533, 536, 498, 561, 555, 532, 544, 500, 494, 481, 538, 502, 506, 512, 523, 532, 495, 521, 502, 476, 515, 497, 485, 501, 506, 505, 494, 520, 497, 492, 486, 507, 498, 513, 519, 529, 501, 495, 533, 518, 550, 515, 529, 531, 519, 512, 481, 486, 517] payload_rows_per_dst=[5554, 5594, 5676, 5584, 5474, 5446, 5570, 5536] dup_rows_per_dst=[2626, 2644, 2764, 2594, 2532, 2552, 2746, 2644] combine_rows_per_src=[5554, 5594, 5676, 5584, 5474, 5446, 5570, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159539200 dispatch_meta_bytes=229312 combine_payload_bytes=159539200 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159768512 ingress_bytes=260202496 link_bytes=260202496 token_bytes=319307712 weight_movement_bytes=100663296 total_nvlink_bytes=419971008 weight_to_token_ratio=0.315255 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.979840347e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158867456 dispatch_meta_bytes=228944 combine_payload_bytes=158867456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159096400 ingress_bytes=184033280 link_bytes=184033280 token_bytes=317963856 weight_movement_bytes=25165824 total_nvlink_bytes=343129680 weight_to_token_ratio=0.079147 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400280622e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=229072 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159317712 ingress_bytes=209420288 link_bytes=209420288 token_bytes=318406352 weight_movement_bytes=50331648 total_nvlink_bytes=368738000 weight_to_token_ratio=0.158074 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593446420e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159678464 dispatch_meta_bytes=229568 combine_payload_bytes=159678464 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159908032 ingress_bytes=184844288 link_bytes=184844288 token_bytes=319586496 weight_movement_bytes=25165824 total_nvlink_bytes=344752320 weight_to_token_ratio=0.078745 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406451456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160309248 dispatch_meta_bytes=229128 combine_payload_bytes=160309248 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160538376 ingress_bytes=185475072 link_bytes=185475072 token_bytes=320847624 weight_movement_bytes=25165824 total_nvlink_bytes=346013448 weight_to_token_ratio=0.078435 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411250993e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158908416 dispatch_meta_bytes=227320 combine_payload_bytes=158908416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159135736 ingress_bytes=184074240 link_bytes=184074240 token_bytes=318044152 weight_movement_bytes=25165824 total_nvlink_bytes=343209976 weight_to_token_ratio=0.079127 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400592280e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=229192 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159317832 ingress_bytes=184254464 link_bytes=184254464 token_bytes=318406472 weight_movement_bytes=25165824 total_nvlink_bytes=343572296 weight_to_token_ratio=0.079037 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401963577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44696 payload_rows_remote=39120 dup_rows=20840 dup_rows_remote=18128 dedup_ratio=0.317993 router_tokens_per_expert=[519, 488, 528, 514, 536, 519, 532, 508, 516, 502, 520, 490, 508, 500, 503, 511, 514, 522, 530, 523, 500, 539, 516, 503, 512, 503, 502, 545, 517, 550, 479, 520, 480, 522, 486, 526, 501, 524, 519, 505, 561, 519, 482, 493, 500, 503, 498, 512, 497, 515, 537, 498, 511, 538, 546, 514, 503, 484, 517, 513, 518, 501, 493, 483] payload_rows_per_dst=[5576, 5614, 5630, 5686, 5530, 5768, 5398, 5494] dup_rows_per_dst=[2712, 2572, 2578, 2570, 2596, 2896, 2386, 2530] combine_rows_per_src=[5576, 5614, 5630, 5686, 5530, 5768, 5398, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=39032 dup_rows=21036 dup_rows_remote=18552 dedup_ratio=0.320984 router_tokens_per_expert=[504, 510, 507, 505, 562, 538, 513, 512, 522, 487, 528, 527, 532, 530, 519, 505, 518, 477, 523, 516, 548, 482, 478, 527, 517, 541, 508, 493, 488, 538, 523, 516, 487, 521, 479, 480, 499, 493, 492, 525, 510, 495, 515, 552, 500, 523, 521, 531, 562, 527, 504, 520, 507, 478, 495, 514, 514, 504, 497, 497, 538, 529, 470, 495] payload_rows_per_dst=[5540, 5652, 5568, 5576, 5468, 5628, 5562, 5506] dup_rows_per_dst=[2762, 2648, 2570, 2672, 2484, 2666, 2652, 2582] combine_rows_per_src=[5540, 5652, 5568, 5576, 5468, 5628, 5562, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44530 payload_rows_remote=39012 dup_rows=21006 dup_rows_remote=18434 dedup_ratio=0.320526 router_tokens_per_expert=[491, 506, 533, 490, 534, 512, 513, 534, 520, 508, 500, 542, 556, 506, 467, 519, 499, 530, 495, 485, 521, 485, 512, 518, 508, 485, 539, 526, 547, 512, 539, 497, 502, 502, 519, 540, 537, 514, 522, 518, 526, 513, 506, 488, 523, 507, 498, 524, 545, 506, 493, 462, 536, 489, 522, 523, 522, 518, 484, 498, 484, 487, 551, 480] payload_rows_per_dst=[5568, 5558, 5518, 5610, 5652, 5554, 5552, 5518] dup_rows_per_dst=[2658, 2678, 2572, 2696, 2656, 2616, 2600, 2530] combine_rows_per_src=[5568, 5558, 5518, 5610, 5652, 5554, 5552, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38878 dup_rows=21114 dup_rows_remote=18336 dedup_ratio=0.322174 router_tokens_per_expert=[521, 521, 530, 518, 508, 534, 495, 463, 497, 540, 510, 505, 524, 522, 510, 529, 534, 536, 467, 541, 480, 523, 485, 514, 523, 501, 510, 465, 506, 504, 541, 513, 548, 515, 513, 548, 524, 492, 490, 493, 468, 483, 512, 492, 493, 526, 495, 517, 563, 506, 522, 523, 524, 469, 523, 531, 525, 473, 497, 525, 501, 510, 576, 521] payload_rows_per_dst=[5596, 5624, 5554, 5472, 5526, 5510, 5544, 5596] dup_rows_per_dst=[2584, 2650, 2606, 2654, 2720, 2462, 2778, 2660] combine_rows_per_src=[5596, 5624, 5554, 5472, 5526, 5510, 5544, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160235520 dispatch_meta_bytes=228992 combine_payload_bytes=160235520 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160464512 ingress_bytes=235732992 link_bytes=235732992 token_bytes=320700032 weight_movement_bytes=75497472 total_nvlink_bytes=396197504 weight_to_token_ratio=0.235415 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.793655695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38854 dup_rows=21118 dup_rows_remote=18390 dedup_ratio=0.322235 router_tokens_per_expert=[529, 487, 527, 507, 520, 517, 506, 520, 542, 510, 489, 522, 482, 521, 483, 547, 520, 474, 512, 488, 492, 492, 492, 511, 518, 501, 467, 540, 547, 491, 493, 521, 511, 509, 510, 500, 515, 490, 550, 546, 518, 532, 513, 531, 490, 496, 542, 524, 519, 505, 529, 545, 552, 501, 538, 523, 510, 490, 508, 487, 510, 487, 518, 501] payload_rows_per_dst=[5588, 5536, 5434, 5540, 5566, 5564, 5668, 5522] dup_rows_per_dst=[2638, 2656, 2528, 2616, 2696, 2728, 2756, 2500] combine_rows_per_src=[5588, 5536, 5434, 5540, 5566, 5564, 5668, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44520 payload_rows_remote=38840 dup_rows=21016 dup_rows_remote=18264 dedup_ratio=0.320679 router_tokens_per_expert=[504, 522, 515, 497, 537, 482, 470, 499, 541, 504, 462, 520, 500, 496, 512, 478, 498, 549, 490, 474, 514, 514, 523, 530, 489, 520, 497, 526, 509, 552, 519, 507, 532, 474, 493, 541, 525, 523, 480, 500, 529, 514, 522, 513, 516, 525, 534, 565, 530, 516, 520, 527, 540, 493, 534, 518, 524, 515, 469, 504, 495, 529, 523, 495] payload_rows_per_dst=[5400, 5398, 5558, 5592, 5556, 5714, 5622, 5680] dup_rows_per_dst=[2514, 2396, 2626, 2512, 2714, 2722, 2780, 2752] combine_rows_per_src=[5400, 5398, 5558, 5592, 5556, 5714, 5622, 5680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38888 dup_rows=21114 dup_rows_remote=18340 dedup_ratio=0.322174 router_tokens_per_expert=[491, 498, 515, 521, 476, 545, 481, 481, 544, 539, 519, 516, 512, 517, 537, 511, 530, 520, 516, 459, 509, 507, 508, 495, 530, 514, 494, 521, 493, 516, 581, 505, 514, 504, 504, 529, 528, 538, 500, 510, 496, 513, 511, 496, 489, 486, 534, 524, 508, 504, 549, 507, 521, 499, 513, 515, 477, 531, 490, 533, 519, 507, 514, 504] payload_rows_per_dst=[5482, 5658, 5544, 5534, 5586, 5584, 5498, 5536] dup_rows_per_dst=[2534, 2732, 2544, 2774, 2668, 2514, 2734, 2614] combine_rows_per_src=[5482, 5658, 5544, 5534, 5586, 5584, 5498, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159875072 dispatch_meta_bytes=230336 combine_payload_bytes=159875072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160105408 ingress_bytes=185040896 link_bytes=185040896 token_bytes=319980480 weight_movement_bytes=25165824 total_nvlink_bytes=345146304 weight_to_token_ratio=0.078648 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407947416e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44520 payload_rows_remote=38986 dup_rows=21016 dup_rows_remote=18452 dedup_ratio=0.320679 router_tokens_per_expert=[523, 544, 506, 513, 545, 497, 505, 519, 495, 478, 543, 504, 509, 520, 475, 525, 503, 509, 524, 534, 491, 518, 460, 526, 505, 516, 509, 511, 509, 525, 501, 514, 494, 507, 506, 512, 530, 538, 499, 510, 504, 490, 498, 514, 524, 534, 528, 528, 519, 509, 507, 504, 518, 495, 479, 498, 552, 542, 541, 492, 512, 498, 520, 510] payload_rows_per_dst=[5596, 5534, 5506, 5574, 5546, 5530, 5568, 5666] dup_rows_per_dst=[2708, 2564, 2624, 2606, 2646, 2710, 2490, 2668] combine_rows_per_src=[5596, 5534, 5506, 5574, 5546, 5530, 5568, 5666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=228856 combine_payload_bytes=159244288 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159473144 ingress_bytes=209575936 link_bytes=209575936 token_bytes=318717432 weight_movement_bytes=50331648 total_nvlink_bytes=369049080 weight_to_token_ratio=0.157919 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594630721e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159793152 dispatch_meta_bytes=229784 combine_payload_bytes=159793152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160022936 ingress_bytes=184958976 link_bytes=184958976 token_bytes=319816088 weight_movement_bytes=25165824 total_nvlink_bytes=344981912 weight_to_token_ratio=0.078688 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407324099e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159145984 dispatch_meta_bytes=228976 combine_payload_bytes=159145984 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159374960 ingress_bytes=209477632 link_bytes=209477632 token_bytes=318520944 weight_movement_bytes=50331648 total_nvlink_bytes=368852592 weight_to_token_ratio=0.158017 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593882741e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=228416 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159317056 ingress_bytes=184254464 link_bytes=184254464 token_bytes=318405696 weight_movement_bytes=25165824 total_nvlink_bytes=343571520 weight_to_token_ratio=0.079037 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401963577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=228912 combine_payload_bytes=159285248 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159514160 ingress_bytes=209616896 link_bytes=209616896 token_bytes=318799408 weight_movement_bytes=50331648 total_nvlink_bytes=369131056 weight_to_token_ratio=0.157879 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594942380e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159686656 dispatch_meta_bytes=229752 combine_payload_bytes=159686656 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159916408 ingress_bytes=210018304 link_bytes=210018304 token_bytes=319603064 weight_movement_bytes=50331648 total_nvlink_bytes=369934712 weight_to_token_ratio=0.157482 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597996631e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44576 payload_rows_remote=38908 dup_rows=20960 dup_rows_remote=18260 dedup_ratio=0.319824 router_tokens_per_expert=[528, 505, 497, 544, 530, 505, 525, 539, 526, 493, 498, 510, 536, 496, 539, 528, 518, 504, 484, 510, 537, 506, 507, 492, 496, 489, 516, 503, 540, 517, 519, 485, 546, 526, 479, 472, 491, 518, 541, 453, 521, 545, 532, 509, 551, 486, 525, 505, 493, 490, 557, 519, 469, 497, 526, 492, 508, 502, 532, 486, 523, 489, 520, 543] payload_rows_per_dst=[5668, 5660, 5792, 5500, 5502, 5480, 5370, 5604] dup_rows_per_dst=[2700, 2592, 2992, 2608, 2550, 2554, 2362, 2602] combine_rows_per_src=[5668, 5660, 5792, 5500, 5502, 5480, 5370, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=38864 dup_rows=21086 dup_rows_remote=18432 dedup_ratio=0.321747 router_tokens_per_expert=[499, 520, 496, 504, 535, 497, 519, 521, 531, 516, 518, 498, 518, 522, 498, 483, 522, 497, 513, 502, 490, 503, 530, 522, 546, 509, 505, 496, 526, 487, 500, 532, 507, 515, 528, 541, 477, 491, 532, 529, 519, 507, 520, 509, 481, 501, 511, 522, 494, 503, 519, 486, 548, 543, 512, 496, 512, 550, 518, 525, 534, 493, 492, 498] payload_rows_per_dst=[5528, 5580, 5494, 5582, 5586, 5518, 5580, 5582] dup_rows_per_dst=[2654, 2588, 2664, 2620, 2654, 2622, 2622, 2662] combine_rows_per_src=[5528, 5580, 5494, 5582, 5586, 5518, 5580, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=39030 dup_rows=20946 dup_rows_remote=18278 dedup_ratio=0.319611 router_tokens_per_expert=[491, 491, 563, 505, 505, 522, 497, 500, 519, 491, 524, 534, 478, 534, 504, 523, 520, 517, 504, 521, 541, 543, 499, 487, 526, 504, 509, 487, 485, 502, 493, 503, 492, 527, 513, 533, 492, 539, 485, 470, 543, 528, 491, 497, 537, 544, 492, 494, 487, 500, 507, 556, 533, 491, 510, 530, 533, 500, 520, 534, 516, 520, 486, 546] payload_rows_per_dst=[5558, 5566, 5614, 5516, 5484, 5706, 5560, 5586] dup_rows_per_dst=[2590, 2648, 2650, 2502, 2618, 2546, 2668, 2724] combine_rows_per_src=[5558, 5566, 5614, 5516, 5484, 5706, 5560, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159367168 dispatch_meta_bytes=228672 combine_payload_bytes=159367168 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159595840 ingress_bytes=234864640 link_bytes=234864640 token_bytes=318963008 weight_movement_bytes=75497472 total_nvlink_bytes=394460480 weight_to_token_ratio=0.236697 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787048539e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38840 dup_rows=21120 dup_rows_remote=18408 dedup_ratio=0.322266 router_tokens_per_expert=[500, 509, 526, 532, 509, 508, 521, 490, 498, 482, 513, 526, 466, 501, 492, 562, 534, 488, 516, 525, 533, 539, 550, 532, 503, 519, 482, 471, 455, 528, 516, 512, 486, 538, 474, 482, 546, 483, 530, 507, 515, 519, 556, 543, 536, 527, 477, 498, 526, 501, 499, 517, 504, 519, 498, 541, 483, 509, 513, 499, 539, 506, 526, 533] payload_rows_per_dst=[5600, 5514, 5624, 5446, 5506, 5618, 5532, 5576] dup_rows_per_dst=[2560, 2566, 2674, 2526, 2630, 2672, 2780, 2712] combine_rows_per_src=[5600, 5514, 5624, 5446, 5506, 5618, 5532, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159186944 dispatch_meta_bytes=229184 combine_payload_bytes=159186944 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159416128 ingress_bytes=184352768 link_bytes=184352768 token_bytes=318603072 weight_movement_bytes=25165824 total_nvlink_bytes=343768896 weight_to_token_ratio=0.078988 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402711557e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38900 dup_rows=21100 dup_rows_remote=18416 dedup_ratio=0.321960 router_tokens_per_expert=[498, 537, 503, 502, 555, 533, 510, 475, 492, 484, 503, 534, 547, 523, 520, 553, 496, 465, 500, 531, 504, 544, 517, 511, 487, 518, 522, 512, 528, 507, 507, 525, 523, 478, 574, 506, 463, 557, 514, 496, 501, 530, 508, 556, 489, 514, 472, 540, 490, 477, 516, 506, 525, 522, 534, 470, 508, 489, 492, 537, 504, 531, 524, 479] payload_rows_per_dst=[5532, 5648, 5542, 5544, 5590, 5536, 5514, 5530] dup_rows_per_dst=[2694, 2664, 2594, 2668, 2632, 2684, 2566, 2598] combine_rows_per_src=[5532, 5648, 5542, 5544, 5590, 5536, 5514, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38816 dup_rows=21124 dup_rows_remote=18502 dedup_ratio=0.322327 router_tokens_per_expert=[522, 492, 528, 521, 495, 521, 520, 501, 505, 477, 537, 489, 515, 498, 530, 494, 516, 525, 523, 531, 523, 543, 509, 505, 494, 561, 485, 504, 481, 542, 536, 506, 508, 497, 545, 518, 509, 509, 541, 520, 470, 485, 522, 522, 509, 515, 535, 515, 566, 525, 506, 488, 524, 512, 485, 533, 508, 467, 523, 484, 474, 557, 471, 496] payload_rows_per_dst=[5544, 5504, 5650, 5596, 5620, 5526, 5548, 5424] dup_rows_per_dst=[2656, 2586, 2700, 2622, 2674, 2620, 2730, 2536] combine_rows_per_src=[5544, 5504, 5650, 5596, 5620, 5526, 5548, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=39002 dup_rows=21026 dup_rows_remote=18388 dedup_ratio=0.320831 router_tokens_per_expert=[530, 518, 507, 545, 524, 494, 491, 508, 492, 517, 523, 524, 477, 548, 497, 508, 506, 488, 519, 493, 522, 529, 511, 505, 533, 515, 497, 519, 539, 484, 541, 536, 500, 515, 510, 516, 481, 489, 532, 505, 519, 508, 543, 475, 529, 463, 514, 506, 500, 506, 550, 511, 473, 498, 535, 523, 501, 487, 508, 522, 515, 522, 525, 547] payload_rows_per_dst=[5570, 5582, 5508, 5622, 5554, 5562, 5518, 5594] dup_rows_per_dst=[2664, 2590, 2638, 2706, 2542, 2552, 2674, 2660] combine_rows_per_src=[5570, 5582, 5508, 5622, 5554, 5562, 5518, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=38972 dup_rows=21014 dup_rows_remote=18340 dedup_ratio=0.320648 router_tokens_per_expert=[535, 526, 529, 524, 511, 497, 530, 535, 489, 544, 515, 511, 521, 495, 515, 522, 517, 533, 524, 543, 496, 508, 503, 490, 517, 518, 480, 513, 485, 496, 515, 516, 533, 524, 520, 494, 490, 529, 514, 500, 520, 538, 487, 537, 465, 493, 501, 514, 521, 519, 513, 537, 505, 511, 505, 526, 532, 484, 513, 483, 489, 511, 508, 499] payload_rows_per_dst=[5632, 5550, 5522, 5528, 5536, 5690, 5586, 5478] dup_rows_per_dst=[2742, 2674, 2706, 2552, 2672, 2420, 2688, 2560] combine_rows_per_src=[5632, 5550, 5522, 5528, 5536, 5690, 5586, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=229232 combine_payload_bytes=159866880 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160096112 ingress_bytes=210198528 link_bytes=210198528 token_bytes=319962992 weight_movement_bytes=50331648 total_nvlink_bytes=370294640 weight_to_token_ratio=0.157305 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599367927e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=228992 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159317632 ingress_bytes=184254464 link_bytes=184254464 token_bytes=318406272 weight_movement_bytes=25165824 total_nvlink_bytes=343572096 weight_to_token_ratio=0.079037 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401963577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158990336 dispatch_meta_bytes=229272 combine_payload_bytes=158990336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159219608 ingress_bytes=184156160 link_bytes=184156160 token_bytes=318209944 weight_movement_bytes=25165824 total_nvlink_bytes=343375768 weight_to_token_ratio=0.079086 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401215597e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=229264 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159563664 ingress_bytes=209666048 link_bytes=209666048 token_bytes=318898064 weight_movement_bytes=50331648 total_nvlink_bytes=369229712 weight_to_token_ratio=0.157830 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595316370e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159752192 dispatch_meta_bytes=229560 combine_payload_bytes=159752192 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159981752 ingress_bytes=260415488 link_bytes=260415488 token_bytes=319733944 weight_movement_bytes=100663296 total_nvlink_bytes=420397240 weight_to_token_ratio=0.314835 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.981460970e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=229248 combine_payload_bytes=159629312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159858560 ingress_bytes=184795136 link_bytes=184795136 token_bytes=319487872 weight_movement_bytes=25165824 total_nvlink_bytes=344653696 weight_to_token_ratio=0.078769 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406077466e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38888 dup_rows=21118 dup_rows_remote=18636 dedup_ratio=0.322235 router_tokens_per_expert=[491, 496, 482, 498, 516, 505, 536, 482, 489, 523, 504, 485, 497, 466, 499, 518, 532, 521, 475, 560, 463, 538, 522, 524, 503, 492, 503, 555, 520, 497, 506, 524, 492, 536, 538, 543, 488, 499, 538, 485, 512, 500, 519, 516, 530, 534, 510, 511, 497, 534, 526, 511, 507, 555, 481, 527, 519, 525, 519, 504, 532, 530, 524, 504] payload_rows_per_dst=[5530, 5544, 5414, 5714, 5524, 5672, 5400, 5620] dup_rows_per_dst=[2482, 2732, 2542, 2792, 2714, 2592, 2570, 2694] combine_rows_per_src=[5530, 5544, 5414, 5714, 5524, 5672, 5400, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44704 payload_rows_remote=39066 dup_rows=20832 dup_rows_remote=18134 dedup_ratio=0.317871 router_tokens_per_expert=[553, 508, 482, 522, 504, 489, 481, 501, 529, 517, 525, 524, 563, 478, 510, 481, 505, 507, 495, 490, 517, 525, 540, 518, 473, 518, 499, 516, 510, 497, 535, 508, 524, 517, 525, 517, 513, 521, 501, 550, 496, 501, 528, 506, 508, 523, 497, 524, 506, 524, 522, 536, 509, 543, 471, 526, 547, 510, 489, 515, 514, 507, 495, 483] payload_rows_per_dst=[5516, 5624, 5596, 5564, 5638, 5600, 5634, 5532] dup_rows_per_dst=[2564, 2630, 2598, 2548, 2698, 2566, 2640, 2588] combine_rows_per_src=[5516, 5624, 5596, 5564, 5638, 5600, 5634, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44522 payload_rows_remote=38964 dup_rows=21014 dup_rows_remote=18266 dedup_ratio=0.320648 router_tokens_per_expert=[466, 531, 503, 516, 514, 541, 498, 520, 498, 520, 517, 507, 484, 526, 512, 514, 545, 497, 543, 503, 501, 493, 497, 466, 523, 520, 500, 538, 499, 477, 486, 465, 534, 485, 530, 550, 516, 504, 495, 521, 543, 475, 541, 509, 532, 503, 481, 491, 566, 510, 516, 506, 480, 501, 535, 539, 530, 539, 521, 532, 510, 528, 517, 508] payload_rows_per_dst=[5570, 5556, 5508, 5536, 5618, 5532, 5558, 5644] dup_rows_per_dst=[2608, 2600, 2582, 2480, 2652, 2618, 2748, 2726] combine_rows_per_src=[5570, 5556, 5508, 5536, 5618, 5532, 5558, 5644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159285248 dispatch_meta_bytes=230096 combine_payload_bytes=159285248 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159515344 ingress_bytes=209616896 link_bytes=209616896 token_bytes=318800592 weight_movement_bytes=50331648 total_nvlink_bytes=369132240 weight_to_token_ratio=0.157878 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594942380e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=228800 combine_payload_bytes=160014336 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160243136 ingress_bytes=235511808 link_bytes=235511808 token_bytes=320257472 weight_movement_bytes=75497472 total_nvlink_bytes=395754944 weight_to_token_ratio=0.235740 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791972740e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44588 payload_rows_remote=38976 dup_rows=20948 dup_rows_remote=18350 dedup_ratio=0.319641 router_tokens_per_expert=[522, 542, 513, 484, 543, 521, 500, 534, 520, 522, 513, 485, 477, 514, 495, 488, 501, 509, 542, 505, 538, 515, 515, 540, 490, 492, 538, 490, 536, 545, 509, 537, 518, 502, 505, 490, 513, 465, 488, 503, 537, 491, 490, 491, 527, 501, 525, 541, 523, 508, 531, 494, 492, 537, 524, 517, 553, 505, 494, 546, 516, 484, 495, 487] payload_rows_per_dst=[5648, 5474, 5620, 5590, 5396, 5648, 5600, 5612] dup_rows_per_dst=[2670, 2554, 2692, 2684, 2422, 2718, 2610, 2598] combine_rows_per_src=[5648, 5474, 5620, 5590, 5396, 5648, 5600, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38960 dup_rows=21058 dup_rows_remote=18544 dedup_ratio=0.321320 router_tokens_per_expert=[528, 498, 523, 508, 513, 464, 526, 513, 500, 504, 510, 517, 523, 500, 500, 484, 488, 504, 539, 540, 539, 530, 509, 501, 512, 494, 492, 496, 524, 499, 517, 482, 500, 528, 515, 527, 534, 506, 542, 525, 540, 466, 492, 502, 526, 516, 493, 541, 540, 510, 570, 508, 509, 497, 517, 505, 531, 503, 526, 506, 532, 480, 494, 510] payload_rows_per_dst=[5524, 5506, 5580, 5518, 5608, 5506, 5666, 5570] dup_rows_per_dst=[2622, 2570, 2720, 2514, 2746, 2646, 2646, 2594] combine_rows_per_src=[5524, 5506, 5580, 5518, 5608, 5506, 5666, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44612 payload_rows_remote=39070 dup_rows=20924 dup_rows_remote=18330 dedup_ratio=0.319275 router_tokens_per_expert=[521, 501, 496, 500, 514, 543, 498, 523, 504, 491, 538, 529, 494, 495, 499, 547, 521, 524, 511, 511, 500, 506, 533, 501, 519, 493, 475, 517, 482, 523, 560, 544, 495, 506, 522, 457, 492, 501, 538, 504, 535, 500, 482, 522, 518, 530, 469, 512, 521, 537, 479, 555, 506, 502, 513, 532, 519, 539, 526, 518, 511, 504, 504, 506] payload_rows_per_dst=[5532, 5598, 5610, 5568, 5512, 5542, 5676, 5574] dup_rows_per_dst=[2660, 2596, 2604, 2658, 2518, 2594, 2614, 2680] combine_rows_per_src=[5532, 5598, 5610, 5568, 5512, 5542, 5676, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38876 dup_rows=21028 dup_rows_remote=18370 dedup_ratio=0.320862 router_tokens_per_expert=[477, 526, 489, 471, 503, 479, 525, 474, 513, 520, 521, 492, 533, 510, 487, 508, 551, 549, 495, 515, 523, 501, 541, 470, 511, 511, 503, 520, 524, 537, 488, 504, 527, 519, 522, 506, 502, 533, 541, 532, 498, 574, 526, 511, 486, 507, 534, 498, 508, 525, 551, 472, 512, 487, 491, 514, 558, 513, 517, 508, 492, 518, 531, 484] payload_rows_per_dst=[5408, 5654, 5632, 5484, 5624, 5656, 5526, 5524] dup_rows_per_dst=[2480, 2514, 2658, 2712, 2740, 2612, 2594, 2718] combine_rows_per_src=[5408, 5654, 5632, 5484, 5624, 5656, 5526, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44376 payload_rows_remote=38834 dup_rows=21160 dup_rows_remote=18558 dedup_ratio=0.322876 router_tokens_per_expert=[541, 512, 547, 489, 505, 503, 528, 483, 538, 487, 471, 525, 499, 496, 519, 537, 507, 520, 511, 511, 488, 471, 498, 531, 553, 520, 531, 485, 507, 555, 550, 507, 531, 509, 518, 519, 506, 536, 517, 526, 532, 551, 511, 505, 472, 466, 529, 509, 492, 499, 488, 489, 495, 501, 512, 513, 519, 518, 545, 500, 509, 537, 505, 484] payload_rows_per_dst=[5574, 5542, 5502, 5680, 5654, 5458, 5384, 5582] dup_rows_per_dst=[2642, 2602, 2572, 2736, 2670, 2692, 2594, 2652] combine_rows_per_src=[5574, 5542, 5502, 5680, 5654, 5458, 5384, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=228920 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159825464 ingress_bytes=235094016 link_bytes=235094016 token_bytes=319422008 weight_movement_bytes=75497472 total_nvlink_bytes=394919480 weight_to_token_ratio=0.236357 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788793826e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159580160 dispatch_meta_bytes=230016 combine_payload_bytes=159580160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159810176 ingress_bytes=184745984 link_bytes=184745984 token_bytes=319390336 weight_movement_bytes=25165824 total_nvlink_bytes=344556160 weight_to_token_ratio=0.078793 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405703476e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=229304 combine_payload_bytes=159645696 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159875000 ingress_bytes=209977344 link_bytes=209977344 token_bytes=319520696 weight_movement_bytes=50331648 total_nvlink_bytes=369852344 weight_to_token_ratio=0.157522 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597684972e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=160030720 dispatch_meta_bytes=229600 combine_payload_bytes=160030720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160260320 ingress_bytes=185196544 link_bytes=185196544 token_bytes=320291040 weight_movement_bytes=25165824 total_nvlink_bytes=345456864 weight_to_token_ratio=0.078572 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409131717e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159064064 dispatch_meta_bytes=229568 combine_payload_bytes=159064064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159293632 ingress_bytes=184229888 link_bytes=184229888 token_bytes=318357696 weight_movement_bytes=25165824 total_nvlink_bytes=343523520 weight_to_token_ratio=0.079049 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401776582e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44580 payload_rows_remote=38988 dup_rows=20956 dup_rows_remote=18248 dedup_ratio=0.319763 router_tokens_per_expert=[519, 521, 565, 519, 519, 497, 510, 492, 506, 506, 513, 525, 504, 491, 484, 499, 494, 508, 540, 505, 520, 520, 529, 489, 467, 528, 534, 491, 502, 511, 501, 562, 487, 547, 494, 539, 535, 502, 523, 523, 519, 491, 524, 473, 530, 513, 499, 493, 496, 502, 485, 537, 530, 497, 480, 482, 541, 563, 522, 543, 499, 495, 517, 516] payload_rows_per_dst=[5644, 5522, 5574, 5584, 5592, 5542, 5488, 5634] dup_rows_per_dst=[2640, 2534, 2636, 2608, 2708, 2542, 2530, 2758] combine_rows_per_src=[5644, 5522, 5574, 5584, 5592, 5542, 5488, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159236096 dispatch_meta_bytes=228984 combine_payload_bytes=159236096 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159465080 ingress_bytes=209567744 link_bytes=209567744 token_bytes=318701176 weight_movement_bytes=50331648 total_nvlink_bytes=369032824 weight_to_token_ratio=0.157927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594568390e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44520 payload_rows_remote=38826 dup_rows=21016 dup_rows_remote=18192 dedup_ratio=0.320679 router_tokens_per_expert=[507, 563, 501, 522, 538, 537, 547, 488, 497, 489, 474, 532, 512, 548, 503, 507, 498, 483, 524, 501, 522, 483, 527, 493, 505, 493, 513, 511, 523, 517, 501, 532, 503, 470, 483, 501, 533, 542, 512, 513, 491, 498, 515, 531, 496, 544, 491, 508, 506, 482, 540, 537, 523, 486, 507, 498, 502, 536, 543, 492, 545, 521, 523, 505] payload_rows_per_dst=[5694, 5682, 5526, 5676, 5444, 5522, 5538, 5438] dup_rows_per_dst=[2824, 2834, 2536, 2588, 2596, 2626, 2508, 2504] combine_rows_per_src=[5694, 5682, 5526, 5676, 5444, 5522, 5538, 5438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=228944 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159923792 ingress_bytes=235192320 link_bytes=235192320 token_bytes=319618640 weight_movement_bytes=75497472 total_nvlink_bytes=395116112 weight_to_token_ratio=0.236211 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789541805e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159031296 dispatch_meta_bytes=228072 combine_payload_bytes=159031296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159259368 ingress_bytes=184197120 link_bytes=184197120 token_bytes=318290664 weight_movement_bytes=25165824 total_nvlink_bytes=343456488 weight_to_token_ratio=0.079066 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401527255e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44612 payload_rows_remote=39028 dup_rows=20924 dup_rows_remote=18466 dedup_ratio=0.319275 router_tokens_per_expert=[461, 489, 539, 479, 519, 517, 522, 503, 529, 513, 473, 551, 488, 504, 525, 531, 506, 528, 494, 535, 480, 519, 534, 518, 471, 535, 544, 510, 519, 529, 506, 516, 490, 563, 471, 529, 484, 519, 512, 514, 515, 522, 537, 510, 529, 525, 521, 503, 532, 516, 510, 516, 495, 488, 486, 478, 487, 535, 465, 533, 541, 542, 491, 522] payload_rows_per_dst=[5526, 5576, 5584, 5594, 5584, 5614, 5584, 5550] dup_rows_per_dst=[2532, 2652, 2644, 2666, 2580, 2710, 2458, 2682] combine_rows_per_src=[5526, 5576, 5584, 5594, 5584, 5614, 5584, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44386 payload_rows_remote=38900 dup_rows=21150 dup_rows_remote=18514 dedup_ratio=0.322723 router_tokens_per_expert=[492, 498, 456, 532, 504, 558, 509, 527, 528, 485, 498, 526, 515, 529, 458, 508, 458, 531, 521, 492, 536, 490, 560, 537, 505, 537, 496, 486, 500, 481, 541, 515, 525, 519, 501, 489, 510, 498, 497, 514, 481, 522, 503, 522, 514, 515, 516, 504, 505, 492, 498, 535, 488, 529, 515, 504, 554, 520, 575, 516, 532, 533, 483, 550] payload_rows_per_dst=[5526, 5546, 5592, 5486, 5514, 5540, 5522, 5660] dup_rows_per_dst=[2626, 2548, 2658, 2636, 2592, 2614, 2610, 2866] combine_rows_per_src=[5526, 5546, 5592, 5486, 5514, 5540, 5522, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=39056 dup_rows=20916 dup_rows_remote=18152 dedup_ratio=0.319153 router_tokens_per_expert=[491, 479, 496, 517, 482, 519, 493, 519, 494, 498, 503, 541, 476, 525, 517, 536, 476, 494, 496, 538, 508, 504, 477, 506, 483, 536, 514, 540, 503, 504, 529, 521, 493, 499, 499, 533, 536, 530, 505, 523, 500, 518, 491, 534, 505, 495, 535, 482, 499, 551, 543, 526, 523, 510, 539, 523, 547, 518, 494, 531, 509, 522, 551, 489] payload_rows_per_dst=[5482, 5630, 5552, 5616, 5604, 5516, 5656, 5564] dup_rows_per_dst=[2510, 2550, 2446, 2644, 2632, 2598, 2772, 2764] combine_rows_per_src=[5482, 5630, 5552, 5616, 5604, 5516, 5656, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38866 dup_rows=21146 dup_rows_remote=18442 dedup_ratio=0.322662 router_tokens_per_expert=[517, 504, 472, 489, 497, 513, 480, 496, 541, 522, 491, 557, 550, 506, 519, 536, 519, 505, 552, 484, 499, 491, 511, 523, 528, 526, 500, 475, 500, 499, 504, 524, 514, 504, 516, 541, 501, 509, 483, 517, 527, 525, 504, 492, 534, 488, 540, 504, 509, 506, 506, 530, 534, 536, 506, 535, 521, 517, 475, 494, 528, 514, 507, 521] payload_rows_per_dst=[5412, 5638, 5578, 5512, 5574, 5524, 5604, 5548] dup_rows_per_dst=[2524, 2806, 2590, 2600, 2596, 2704, 2720, 2606] combine_rows_per_src=[5412, 5638, 5578, 5512, 5574, 5524, 5604, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44572 payload_rows_remote=39074 dup_rows=20964 dup_rows_remote=18328 dedup_ratio=0.319885 router_tokens_per_expert=[548, 542, 489, 504, 505, 523, 521, 541, 507, 514, 490, 525, 479, 506, 518, 528, 476, 527, 542, 505, 533, 473, 530, 493, 500, 501, 476, 512, 528, 505, 494, 480, 536, 502, 498, 506, 539, 533, 523, 533, 506, 535, 471, 547, 538, 523, 495, 490, 501, 482, 518, 519, 472, 499, 495, 531, 515, 521, 547, 518, 526, 501, 491, 542] payload_rows_per_dst=[5648, 5498, 5590, 5556, 5660, 5534, 5502, 5584] dup_rows_per_dst=[2698, 2636, 2568, 2436, 2680, 2676, 2532, 2738] combine_rows_per_src=[5648, 5498, 5590, 5556, 5660, 5534, 5502, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44452 payload_rows_remote=38862 dup_rows=21084 dup_rows_remote=18478 dedup_ratio=0.321716 router_tokens_per_expert=[524, 465, 519, 521, 509, 492, 509, 537, 492, 514, 500, 489, 517, 497, 511, 533, 506, 509, 502, 487, 488, 571, 524, 511, 560, 520, 538, 494, 491, 521, 495, 532, 482, 506, 501, 550, 525, 551, 490, 544, 504, 550, 507, 554, 491, 488, 508, 517, 534, 498, 527, 475, 511, 497, 489, 509, 511, 511, 542, 530, 513, 501, 490, 484] payload_rows_per_dst=[5510, 5516, 5590, 5516, 5628, 5578, 5478, 5636] dup_rows_per_dst=[2642, 2590, 2606, 2786, 2670, 2660, 2602, 2528] combine_rows_per_src=[5510, 5516, 5590, 5516, 5628, 5578, 5478, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=229656 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159564056 ingress_bytes=184500224 link_bytes=184500224 token_bytes=318898456 weight_movement_bytes=25165824 total_nvlink_bytes=344064280 weight_to_token_ratio=0.078915 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403833526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159858688 dispatch_meta_bytes=229976 combine_payload_bytes=159858688 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160088664 ingress_bytes=185024512 link_bytes=185024512 token_bytes=319947352 weight_movement_bytes=25165824 total_nvlink_bytes=345113176 weight_to_token_ratio=0.078656 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407822752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159973376 dispatch_meta_bytes=228832 combine_payload_bytes=159973376 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=160202208 ingress_bytes=260636672 link_bytes=260636672 token_bytes=320175584 weight_movement_bytes=100663296 total_nvlink_bytes=420838880 weight_to_token_ratio=0.314400 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.983143925e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159195136 dispatch_meta_bytes=229232 combine_payload_bytes=159195136 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159424368 ingress_bytes=209526784 link_bytes=209526784 token_bytes=318619504 weight_movement_bytes=50331648 total_nvlink_bytes=368951152 weight_to_token_ratio=0.157968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594256731e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44462 payload_rows_remote=38956 dup_rows=21074 dup_rows_remote=18496 dedup_ratio=0.321564 router_tokens_per_expert=[513, 499, 564, 529, 497, 560, 498, 521, 513, 482, 522, 498, 531, 502, 491, 503, 474, 508, 509, 533, 496, 537, 508, 492, 489, 510, 535, 477, 507, 523, 544, 547, 526, 516, 472, 520, 493, 521, 503, 491, 518, 529, 500, 479, 521, 516, 512, 517, 496, 501, 520, 490, 534, 564, 528, 515, 514, 517, 504, 535, 497, 479, 495, 533] payload_rows_per_dst=[5608, 5518, 5568, 5664, 5506, 5494, 5572, 5532] dup_rows_per_dst=[2754, 2566, 2546, 2600, 2578, 2690, 2724, 2616] combine_rows_per_src=[5608, 5518, 5568, 5664, 5506, 5494, 5572, 5532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=160047104 dispatch_meta_bytes=229608 combine_payload_bytes=160047104 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160276712 ingress_bytes=185212928 link_bytes=185212928 token_bytes=320323816 weight_movement_bytes=25165824 total_nvlink_bytes=345489640 weight_to_token_ratio=0.078564 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409256380e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=38808 dup_rows=20968 dup_rows_remote=18112 dedup_ratio=0.319946 router_tokens_per_expert=[514, 557, 483, 491, 479, 530, 543, 522, 479, 472, 486, 503, 541, 476, 521, 536, 514, 485, 509, 509, 512, 511, 485, 482, 538, 499, 540, 519, 491, 513, 540, 505, 528, 487, 525, 511, 539, 546, 505, 545, 560, 558, 498, 495, 507, 498, 500, 479, 508, 503, 529, 519, 506, 518, 510, 502, 548, 494, 516, 480, 541, 508, 499, 521] payload_rows_per_dst=[5760, 5632, 5634, 5598, 5474, 5576, 5670, 5224] dup_rows_per_dst=[2856, 2662, 2694, 2698, 2448, 2608, 2808, 2194] combine_rows_per_src=[5760, 5632, 5634, 5598, 5474, 5576, 5670, 5224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159178752 dispatch_meta_bytes=229360 combine_payload_bytes=159178752 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159408112 ingress_bytes=184344576 link_bytes=184344576 token_bytes=318586864 weight_movement_bytes=25165824 total_nvlink_bytes=343752688 weight_to_token_ratio=0.078992 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402649225e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=229808 combine_payload_bytes=159563776 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159793584 ingress_bytes=235061248 link_bytes=235061248 token_bytes=319357360 weight_movement_bytes=75497472 total_nvlink_bytes=394854832 weight_to_token_ratio=0.236404 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788544499e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158957568 dispatch_meta_bytes=227680 combine_payload_bytes=158957568 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159185248 ingress_bytes=184123392 link_bytes=184123392 token_bytes=318142816 weight_movement_bytes=25165824 total_nvlink_bytes=343308640 weight_to_token_ratio=0.079102 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400966270e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38974 dup_rows=21058 dup_rows_remote=18496 dedup_ratio=0.321320 router_tokens_per_expert=[480, 481, 520, 490, 514, 529, 520, 514, 505, 519, 550, 482, 487, 517, 520, 488, 500, 522, 572, 496, 509, 525, 505, 518, 493, 503, 525, 471, 525, 541, 463, 512, 496, 480, 510, 503, 519, 523, 487, 526, 553, 533, 535, 484, 528, 532, 498, 528, 500, 502, 519, 518, 530, 526, 503, 523, 513, 517, 492, 520, 520, 520, 537, 497] payload_rows_per_dst=[5562, 5494, 5624, 5504, 5514, 5592, 5574, 5614] dup_rows_per_dst=[2534, 2642, 2670, 2562, 2574, 2790, 2668, 2618] combine_rows_per_src=[5562, 5494, 5624, 5504, 5514, 5592, 5574, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44374 payload_rows_remote=38804 dup_rows=21162 dup_rows_remote=18562 dedup_ratio=0.322906 router_tokens_per_expert=[517, 552, 525, 517, 525, 494, 511, 506, 503, 501, 520, 559, 478, 550, 552, 515, 514, 522, 515, 528, 541, 509, 516, 467, 507, 507, 516, 518, 522, 485, 526, 504, 487, 531, 533, 523, 499, 493, 527, 528, 522, 491, 476, 511, 500, 482, 491, 470, 551, 475, 474, 516, 489, 519, 514, 547, 503, 484, 491, 521, 519, 538, 508, 533] payload_rows_per_dst=[5580, 5620, 5546, 5450, 5590, 5428, 5570, 5590] dup_rows_per_dst=[2714, 2736, 2678, 2720, 2652, 2458, 2600, 2604] combine_rows_per_src=[5580, 5620, 5546, 5450, 5590, 5428, 5570, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38988 dup_rows=21002 dup_rows_remote=18332 dedup_ratio=0.320465 router_tokens_per_expert=[526, 485, 518, 526, 520, 495, 508, 497, 467, 502, 507, 501, 488, 489, 495, 512, 518, 556, 485, 502, 521, 513, 531, 514, 537, 545, 486, 545, 518, 530, 484, 523, 526, 509, 531, 523, 491, 522, 513, 509, 521, 475, 486, 540, 527, 486, 476, 513, 502, 527, 508, 521, 509, 537, 550, 543, 477, 493, 503, 512, 516, 524, 521, 533] payload_rows_per_dst=[5510, 5426, 5494, 5660, 5648, 5522, 5728, 5546] dup_rows_per_dst=[2640, 2496, 2578, 2676, 2600, 2562, 2780, 2670] combine_rows_per_src=[5510, 5426, 5494, 5660, 5648, 5522, 5728, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44714 payload_rows_remote=39092 dup_rows=20822 dup_rows_remote=18168 dedup_ratio=0.317719 router_tokens_per_expert=[479, 549, 541, 513, 482, 542, 497, 513, 519, 511, 488, 536, 523, 488, 541, 496, 538, 530, 507, 481, 518, 517, 540, 551, 490, 483, 483, 528, 503, 514, 526, 498, 482, 482, 505, 482, 530, 522, 514, 518, 511, 510, 494, 531, 566, 523, 469, 534, 506, 501, 501, 531, 486, 534, 499, 501, 517, 517, 495, 508, 507, 500, 533, 534] payload_rows_per_dst=[5580, 5600, 5716, 5576, 5546, 5622, 5546, 5528] dup_rows_per_dst=[2652, 2604, 2648, 2474, 2524, 2654, 2572, 2694] combine_rows_per_src=[5580, 5600, 5716, 5576, 5546, 5622, 5546, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38864 dup_rows=21028 dup_rows_remote=18326 dedup_ratio=0.320862 router_tokens_per_expert=[496, 514, 516, 532, 517, 499, 522, 546, 552, 519, 523, 521, 505, 521, 503, 529, 517, 521, 535, 499, 504, 530, 514, 515, 493, 530, 506, 491, 507, 507, 516, 493, 523, 524, 492, 538, 499, 519, 514, 547, 509, 468, 515, 515, 519, 512, 522, 515, 494, 443, 503, 511, 480, 536, 513, 441, 526, 542, 471, 553, 539, 488, 499, 505] payload_rows_per_dst=[5624, 5644, 5556, 5624, 5544, 5524, 5368, 5624] dup_rows_per_dst=[2660, 2702, 2714, 2462, 2768, 2626, 2474, 2622] combine_rows_per_src=[5624, 5644, 5556, 5624, 5544, 5524, 5368, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44452 payload_rows_remote=38972 dup_rows=21084 dup_rows_remote=18574 dedup_ratio=0.321716 router_tokens_per_expert=[520, 475, 465, 494, 505, 521, 527, 514, 516, 510, 488, 511, 545, 526, 509, 509, 450, 490, 509, 480, 528, 505, 509, 524, 488, 507, 532, 531, 527, 472, 559, 501, 515, 516, 565, 538, 514, 529, 537, 521, 540, 486, 510, 524, 543, 499, 496, 527, 497, 511, 474, 525, 515, 520, 521, 490, 535, 504, 502, 532, 532, 500, 520, 483] payload_rows_per_dst=[5454, 5574, 5480, 5560, 5682, 5554, 5570, 5578] dup_rows_per_dst=[2588, 2654, 2510, 2674, 2788, 2696, 2536, 2638] combine_rows_per_src=[5454, 5574, 5480, 5560, 5682, 5554, 5570, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159637504 dispatch_meta_bytes=229880 combine_payload_bytes=159637504 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159867384 ingress_bytes=184803328 link_bytes=184803328 token_bytes=319504888 weight_movement_bytes=25165824 total_nvlink_bytes=344670712 weight_to_token_ratio=0.078765 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406139798e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=229464 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159170648 ingress_bytes=184107008 link_bytes=184107008 token_bytes=318111832 weight_movement_bytes=25165824 total_nvlink_bytes=343277656 weight_to_token_ratio=0.079110 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400841607e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44396 payload_rows_remote=38898 dup_rows=21140 dup_rows_remote=18602 dedup_ratio=0.322571 router_tokens_per_expert=[523, 466, 532, 491, 510, 522, 532, 507, 504, 479, 490, 530, 502, 505, 515, 501, 511, 515, 488, 532, 468, 489, 506, 513, 526, 517, 520, 480, 537, 506, 552, 509, 494, 487, 512, 512, 522, 486, 485, 520, 555, 487, 561, 527, 504, 527, 486, 507, 471, 526, 530, 537, 511, 530, 481, 565, 525, 490, 507, 519, 550, 550, 518, 508] payload_rows_per_dst=[5546, 5496, 5502, 5514, 5498, 5632, 5620, 5588] dup_rows_per_dst=[2620, 2556, 2542, 2780, 2538, 2676, 2682, 2746] combine_rows_per_src=[5546, 5496, 5502, 5514, 5498, 5632, 5620, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=229280 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159924128 ingress_bytes=184860672 link_bytes=184860672 token_bytes=319618976 weight_movement_bytes=25165824 total_nvlink_bytes=344784800 weight_to_token_ratio=0.078737 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406576119e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160120832 dispatch_meta_bytes=229040 combine_payload_bytes=160120832 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160349872 ingress_bytes=185286656 link_bytes=185286656 token_bytes=320470704 weight_movement_bytes=25165824 total_nvlink_bytes=345636528 weight_to_token_ratio=0.078528 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409817365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159186944 dispatch_meta_bytes=228760 combine_payload_bytes=159186944 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159415704 ingress_bytes=184352768 link_bytes=184352768 token_bytes=318602648 weight_movement_bytes=25165824 total_nvlink_bytes=343768472 weight_to_token_ratio=0.078988 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402711557e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44584 payload_rows_remote=39140 dup_rows=20952 dup_rows_remote=18320 dedup_ratio=0.319702 router_tokens_per_expert=[519, 482, 502, 482, 498, 475, 512, 485, 511, 482, 535, 501, 474, 498, 477, 510, 530, 509, 520, 537, 551, 523, 536, 533, 558, 536, 490, 506, 539, 485, 522, 505, 530, 513, 513, 499, 544, 480, 528, 518, 481, 525, 516, 508, 526, 541, 549, 516, 542, 499, 493, 506, 541, 523, 529, 494, 482, 508, 501, 537, 516, 494, 480, 513] payload_rows_per_dst=[5444, 5602, 5568, 5664, 5498, 5622, 5620, 5566] dup_rows_per_dst=[2632, 2658, 2640, 2666, 2524, 2702, 2634, 2496] combine_rows_per_src=[5444, 5602, 5568, 5664, 5498, 5622, 5620, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=230184 combine_payload_bytes=159629312 prefetch_bytes=25165824 reduce_bytes=100663296 egress_bytes=159859496 ingress_bytes=285458432 link_bytes=285458432 token_bytes=319488808 weight_movement_bytes=125829120 total_nvlink_bytes=445317928 weight_to_token_ratio=0.393845 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=2.172008839e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159326208 dispatch_meta_bytes=230000 combine_payload_bytes=159326208 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159556208 ingress_bytes=209657856 link_bytes=209657856 token_bytes=318882416 weight_movement_bytes=50331648 total_nvlink_bytes=369214064 weight_to_token_ratio=0.157838 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595254038e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160317440 dispatch_meta_bytes=229840 combine_payload_bytes=160317440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160547280 ingress_bytes=185483264 link_bytes=185483264 token_bytes=320864720 weight_movement_bytes=25165824 total_nvlink_bytes=346030544 weight_to_token_ratio=0.078431 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411313325e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=38940 dup_rows=21096 dup_rows_remote=18410 dedup_ratio=0.321899 router_tokens_per_expert=[512, 493, 547, 472, 521, 502, 491, 498, 514, 518, 512, 516, 503, 519, 526, 475, 577, 533, 509, 503, 534, 482, 504, 498, 501, 520, 498, 508, 515, 514, 528, 509, 483, 479, 513, 526, 534, 499, 553, 505, 538, 547, 529, 491, 494, 532, 484, 519, 478, 480, 515, 533, 498, 508, 508, 552, 516, 540, 529, 495, 516, 525, 494, 503] payload_rows_per_dst=[5496, 5578, 5592, 5500, 5564, 5598, 5522, 5590] dup_rows_per_dst=[2576, 2588, 2688, 2686, 2620, 2670, 2622, 2646] combine_rows_per_src=[5496, 5578, 5592, 5500, 5564, 5598, 5522, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=39102 dup_rows=20916 dup_rows_remote=18328 dedup_ratio=0.319153 router_tokens_per_expert=[545, 510, 483, 477, 531, 559, 530, 492, 521, 528, 530, 498, 493, 529, 540, 498, 524, 481, 481, 515, 539, 504, 545, 536, 499, 498, 503, 506, 478, 502, 533, 479, 517, 497, 475, 504, 542, 539, 535, 470, 503, 519, 504, 504, 482, 527, 509, 491, 526, 501, 509, 501, 485, 549, 492, 490, 550, 498, 527, 544, 536, 492, 533, 530] payload_rows_per_dst=[5634, 5644, 5588, 5516, 5530, 5538, 5518, 5652] dup_rows_per_dst=[2620, 2630, 2662, 2480, 2628, 2540, 2588, 2768] combine_rows_per_src=[5634, 5644, 5588, 5516, 5530, 5538, 5518, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44482 payload_rows_remote=38802 dup_rows=21054 dup_rows_remote=18288 dedup_ratio=0.321259 router_tokens_per_expert=[507, 521, 507, 528, 526, 525, 504, 530, 481, 483, 521, 497, 526, 480, 489, 505, 503, 508, 514, 544, 520, 520, 505, 529, 541, 449, 529, 516, 514, 517, 505, 516, 533, 449, 539, 520, 534, 484, 485, 512, 515, 539, 543, 530, 557, 503, 565, 537, 473, 484, 488, 534, 490, 481, 492, 499, 543, 535, 521, 515, 498, 507, 512, 491] payload_rows_per_dst=[5578, 5462, 5538, 5546, 5508, 5760, 5410, 5680] dup_rows_per_dst=[2718, 2502, 2524, 2628, 2448, 2996, 2472, 2766] combine_rows_per_src=[5578, 5462, 5538, 5546, 5508, 5760, 5410, 5680] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44328 payload_rows_remote=38846 dup_rows=21208 dup_rows_remote=18714 dedup_ratio=0.323608 router_tokens_per_expert=[501, 513, 523, 509, 526, 490, 519, 515, 478, 501, 500, 497, 520, 515, 510, 467, 518, 499, 524, 515, 515, 521, 501, 492, 528, 497, 534, 511, 539, 539, 520, 471, 552, 533, 500, 511, 527, 528, 518, 495, 530, 523, 503, 500, 532, 554, 504, 526, 482, 527, 483, 489, 489, 549, 511, 504, 524, 515, 497, 524, 508, 513, 509, 500] payload_rows_per_dst=[5646, 5482, 5478, 5620, 5602, 5536, 5478, 5486] dup_rows_per_dst=[2546, 2494, 2692, 2658, 2726, 2808, 2590, 2694] combine_rows_per_src=[5646, 5482, 5478, 5620, 5602, 5536, 5478, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44528 payload_rows_remote=38966 dup_rows=21008 dup_rows_remote=18370 dedup_ratio=0.320557 router_tokens_per_expert=[503, 490, 544, 502, 507, 507, 519, 484, 522, 479, 505, 503, 485, 489, 521, 501, 555, 530, 555, 514, 517, 530, 502, 506, 521, 490, 522, 503, 505, 535, 535, 524, 527, 502, 515, 481, 531, 497, 523, 536, 504, 511, 546, 508, 526, 509, 488, 508, 520, 528, 500, 518, 532, 484, 492, 492, 524, 523, 519, 540, 496, 513, 498, 472] payload_rows_per_dst=[5506, 5510, 5674, 5546, 5594, 5562, 5568, 5568] dup_rows_per_dst=[2606, 2500, 2744, 2724, 2630, 2638, 2564, 2602] combine_rows_per_src=[5506, 5510, 5674, 5546, 5594, 5562, 5568, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44306 payload_rows_remote=38804 dup_rows=21230 dup_rows_remote=18626 dedup_ratio=0.323944 router_tokens_per_expert=[491, 520, 546, 504, 539, 473, 464, 503, 515, 513, 506, 541, 545, 493, 491, 538, 555, 510, 471, 518, 480, 505, 519, 495, 494, 533, 501, 529, 482, 497, 509, 527, 539, 482, 513, 501, 503, 502, 538, 527, 526, 524, 471, 531, 480, 512, 518, 505, 500, 515, 516, 525, 469, 543, 533, 524, 548, 550, 491, 507, 503, 532, 512, 521] payload_rows_per_dst=[5498, 5584, 5502, 5574, 5532, 5524, 5530, 5562] dup_rows_per_dst=[2582, 2700, 2604, 2570, 2678, 2610, 2720, 2766] combine_rows_per_src=[5498, 5584, 5502, 5574, 5532, 5524, 5530, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=229400 combine_payload_bytes=159498240 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159727640 ingress_bytes=209829888 link_bytes=209829888 token_bytes=319225880 weight_movement_bytes=50331648 total_nvlink_bytes=369557528 weight_to_token_ratio=0.157668 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596563003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160161792 dispatch_meta_bytes=229720 combine_payload_bytes=160161792 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160391512 ingress_bytes=185327616 link_bytes=185327616 token_bytes=320553304 weight_movement_bytes=25165824 total_nvlink_bytes=345719128 weight_to_token_ratio=0.078507 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410129024e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44608 payload_rows_remote=39066 dup_rows=20928 dup_rows_remote=18302 dedup_ratio=0.319336 router_tokens_per_expert=[504, 520, 548, 491, 516, 518, 496, 491, 485, 506, 491, 499, 521, 502, 516, 504, 500, 536, 530, 576, 492, 519, 492, 525, 520, 506, 494, 483, 551, 525, 513, 534, 506, 502, 492, 486, 521, 540, 520, 517, 540, 536, 524, 526, 522, 475, 518, 490, 519, 496, 526, 493, 517, 524, 503, 506, 534, 501, 498, 504, 519, 522, 494, 493] payload_rows_per_dst=[5542, 5452, 5654, 5594, 5542, 5634, 5618, 5572] dup_rows_per_dst=[2626, 2596, 2686, 2658, 2626, 2628, 2550, 2558] combine_rows_per_src=[5542, 5452, 5654, 5594, 5542, 5634, 5618, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158932992 dispatch_meta_bytes=228360 combine_payload_bytes=158932992 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159161352 ingress_bytes=209264640 link_bytes=209264640 token_bytes=318094344 weight_movement_bytes=50331648 total_nvlink_bytes=368425992 weight_to_token_ratio=0.158229 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592262118e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44476 payload_rows_remote=38876 dup_rows=21060 dup_rows_remote=18296 dedup_ratio=0.321350 router_tokens_per_expert=[491, 505, 489, 534, 505, 557, 504, 524, 521, 539, 487, 494, 525, 482, 492, 476, 507, 504, 511, 504, 520, 506, 493, 557, 505, 501, 494, 527, 531, 563, 511, 539, 503, 526, 528, 530, 567, 535, 471, 560, 512, 513, 527, 534, 488, 484, 536, 519, 496, 505, 511, 495, 496, 501, 530, 472, 528, 506, 503, 527, 502, 479, 521, 465] payload_rows_per_dst=[5600, 5792, 5618, 5578, 5636, 5220, 5678, 5354] dup_rows_per_dst=[2764, 3018, 2588, 2616, 2804, 2228, 2974, 2068] combine_rows_per_src=[5600, 5792, 5618, 5578, 5636, 5220, 5678, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=230240 combine_payload_bytes=159113216 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159343456 ingress_bytes=184279040 link_bytes=184279040 token_bytes=318456672 weight_movement_bytes=25165824 total_nvlink_bytes=343622496 weight_to_token_ratio=0.079024 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402150572e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159604736 dispatch_meta_bytes=229344 combine_payload_bytes=159604736 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159834080 ingress_bytes=209936384 link_bytes=209936384 token_bytes=319438816 weight_movement_bytes=50331648 total_nvlink_bytes=369770464 weight_to_token_ratio=0.157563 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597373314e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=229720 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159170904 ingress_bytes=234438656 link_bytes=234438656 token_bytes=318112088 weight_movement_bytes=75497472 total_nvlink_bytes=393609560 weight_to_token_ratio=0.237330 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783807293e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=229472 combine_payload_bytes=160014336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160243808 ingress_bytes=185180160 link_bytes=185180160 token_bytes=320258144 weight_movement_bytes=25165824 total_nvlink_bytes=345423968 weight_to_token_ratio=0.078580 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409007054e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159236096 dispatch_meta_bytes=228688 combine_payload_bytes=159236096 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159464784 ingress_bytes=184401920 link_bytes=184401920 token_bytes=318700880 weight_movement_bytes=25165824 total_nvlink_bytes=343866704 weight_to_token_ratio=0.078964 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403085547e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38780 dup_rows=21166 dup_rows_remote=18566 dedup_ratio=0.322968 router_tokens_per_expert=[500, 514, 511, 523, 517, 495, 484, 519, 514, 520, 555, 531, 502, 512, 520, 493, 530, 478, 520, 493, 533, 508, 522, 497, 541, 535, 504, 498, 501, 509, 507, 500, 531, 538, 513, 521, 499, 468, 493, 504, 462, 511, 490, 508, 536, 505, 515, 511, 554, 548, 509, 544, 527, 547, 515, 518, 511, 481, 512, 504, 486, 508, 500, 513] payload_rows_per_dst=[5512, 5542, 5596, 5590, 5478, 5440, 5724, 5488] dup_rows_per_dst=[2614, 2752, 2566, 2600, 2656, 2636, 2800, 2542] combine_rows_per_src=[5512, 5542, 5596, 5590, 5478, 5440, 5724, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44432 payload_rows_remote=38932 dup_rows=21104 dup_rows_remote=18450 dedup_ratio=0.322021 router_tokens_per_expert=[571, 506, 519, 514, 499, 492, 543, 518, 528, 489, 482, 529, 561, 514, 512, 487, 527, 525, 496, 516, 518, 505, 506, 517, 508, 489, 506, 532, 504, 484, 484, 517, 489, 529, 496, 535, 507, 547, 510, 492, 501, 518, 494, 533, 518, 501, 531, 525, 502, 499, 520, 537, 507, 495, 543, 474, 514, 483, 492, 500, 535, 486, 514, 543] payload_rows_per_dst=[5644, 5508, 5530, 5526, 5584, 5606, 5500, 5534] dup_rows_per_dst=[2680, 2696, 2690, 2522, 2626, 2636, 2654, 2600] combine_rows_per_src=[5644, 5508, 5530, 5526, 5584, 5606, 5500, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44390 payload_rows_remote=38820 dup_rows=21146 dup_rows_remote=18362 dedup_ratio=0.322662 router_tokens_per_expert=[479, 509, 530, 487, 547, 496, 503, 499, 482, 543, 541, 476, 515, 506, 541, 523, 543, 533, 461, 496, 531, 489, 520, 505, 530, 534, 502, 498, 506, 520, 543, 500, 564, 477, 520, 497, 493, 509, 510, 525, 557, 508, 533, 547, 507, 482, 493, 537, 486, 520, 516, 511, 525, 508, 490, 514, 485, 505, 513, 541, 506, 485, 506, 510] payload_rows_per_dst=[5356, 5614, 5718, 5468, 5496, 5604, 5564, 5570] dup_rows_per_dst=[2242, 2640, 2940, 2546, 2694, 2656, 2644, 2784] combine_rows_per_src=[5356, 5614, 5718, 5468, 5496, 5604, 5564, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44474 payload_rows_remote=38904 dup_rows=21062 dup_rows_remote=18374 dedup_ratio=0.321381 router_tokens_per_expert=[496, 524, 503, 525, 521, 491, 511, 506, 537, 499, 474, 536, 534, 507, 544, 498, 483, 495, 489, 508, 501, 492, 515, 460, 496, 485, 497, 542, 508, 540, 529, 493, 519, 523, 516, 491, 512, 519, 537, 537, 532, 538, 541, 467, 487, 565, 513, 535, 487, 520, 514, 526, 490, 503, 476, 533, 512, 508, 522, 518, 517, 505, 559, 507] payload_rows_per_dst=[5556, 5570, 5486, 5544, 5578, 5602, 5562, 5576] dup_rows_per_dst=[2598, 2688, 2400, 2636, 2730, 2754, 2536, 2720] combine_rows_per_src=[5556, 5570, 5486, 5544, 5578, 5602, 5562, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44656 payload_rows_remote=39014 dup_rows=20880 dup_rows_remote=18252 dedup_ratio=0.318604 router_tokens_per_expert=[488, 488, 543, 524, 500, 489, 523, 521, 500, 503, 489, 526, 521, 503, 502, 526, 486, 520, 513, 481, 520, 469, 532, 489, 499, 529, 538, 544, 510, 545, 494, 518, 513, 517, 538, 520, 487, 514, 510, 523, 523, 511, 507, 537, 524, 530, 488, 515, 511, 517, 511, 484, 541, 513, 511, 552, 475, 487, 505, 522, 506, 533, 493, 517] payload_rows_per_dst=[5598, 5536, 5550, 5632, 5564, 5642, 5600, 5534] dup_rows_per_dst=[2554, 2604, 2470, 2722, 2680, 2628, 2680, 2542] combine_rows_per_src=[5598, 5536, 5550, 5632, 5564, 5642, 5600, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38958 dup_rows=21046 dup_rows_remote=18490 dedup_ratio=0.321136 router_tokens_per_expert=[534, 484, 551, 513, 516, 513, 538, 545, 523, 548, 569, 516, 493, 505, 506, 493, 502, 475, 497, 555, 483, 530, 497, 505, 496, 488, 508, 504, 523, 507, 485, 533, 570, 520, 483, 515, 503, 486, 504, 532, 481, 506, 461, 517, 474, 544, 519, 468, 488, 515, 527, 499, 540, 493, 505, 526, 513, 563, 501, 553, 506, 531, 478, 512] payload_rows_per_dst=[5652, 5552, 5532, 5580, 5582, 5466, 5554, 5572] dup_rows_per_dst=[2736, 2754, 2556, 2508, 2644, 2474, 2632, 2742] combine_rows_per_src=[5652, 5552, 5532, 5580, 5582, 5466, 5554, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=229384 combine_payload_bytes=158842880 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159072264 ingress_bytes=209174528 link_bytes=209174528 token_bytes=317915144 weight_movement_bytes=50331648 total_nvlink_bytes=368246792 weight_to_token_ratio=0.158318 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591576470e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159465472 dispatch_meta_bytes=229528 combine_payload_bytes=159465472 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159695000 ingress_bytes=209797120 link_bytes=209797120 token_bytes=319160472 weight_movement_bytes=50331648 total_nvlink_bytes=369492120 weight_to_token_ratio=0.157700 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596313676e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44380 payload_rows_remote=38794 dup_rows=21156 dup_rows_remote=18460 dedup_ratio=0.322815 router_tokens_per_expert=[500, 528, 535, 504, 499, 503, 517, 499, 517, 525, 535, 511, 532, 557, 532, 490, 499, 497, 513, 479, 484, 508, 532, 485, 504, 474, 503, 529, 482, 502, 526, 511, 493, 509, 508, 530, 515, 518, 541, 527, 518, 503, 528, 512, 502, 507, 508, 501, 524, 495, 539, 496, 552, 486, 519, 503, 508, 500, 527, 473, 554, 545, 516, 499] payload_rows_per_dst=[5470, 5632, 5498, 5530, 5586, 5534, 5532, 5598] dup_rows_per_dst=[2700, 2766, 2496, 2532, 2696, 2624, 2696, 2646] combine_rows_per_src=[5470, 5632, 5498, 5530, 5586, 5534, 5532, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44526 payload_rows_remote=38774 dup_rows=21010 dup_rows_remote=18136 dedup_ratio=0.320587 router_tokens_per_expert=[538, 510, 514, 544, 510, 567, 501, 515, 502, 485, 517, 495, 503, 494, 495, 508, 530, 473, 513, 505, 489, 527, 485, 559, 503, 518, 551, 551, 523, 536, 494, 528, 510, 473, 544, 493, 511, 492, 500, 494, 519, 539, 510, 483, 520, 508, 497, 518, 477, 495, 517, 524, 518, 483, 548, 511, 550, 510, 517, 494, 507, 514, 515, 494] payload_rows_per_dst=[5752, 5508, 5566, 5686, 5568, 5492, 5442, 5512] dup_rows_per_dst=[2874, 2524, 2596, 2724, 2488, 2638, 2476, 2690] combine_rows_per_src=[5752, 5508, 5566, 5686, 5568, 5492, 5442, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=228728 combine_payload_bytes=159006720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159235448 ingress_bytes=184172544 link_bytes=184172544 token_bytes=318242168 weight_movement_bytes=25165824 total_nvlink_bytes=343407992 weight_to_token_ratio=0.079078 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401340260e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229112 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159579896 ingress_bytes=184516608 link_bytes=184516608 token_bytes=318930680 weight_movement_bytes=25165824 total_nvlink_bytes=344096504 weight_to_token_ratio=0.078907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403958190e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159801344 dispatch_meta_bytes=229064 combine_payload_bytes=159801344 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=160030408 ingress_bytes=260464640 link_bytes=260464640 token_bytes=319831752 weight_movement_bytes=100663296 total_nvlink_bytes=420495048 weight_to_token_ratio=0.314738 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.981834960e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159571968 dispatch_meta_bytes=229792 combine_payload_bytes=159571968 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159801760 ingress_bytes=184737792 link_bytes=184737792 token_bytes=319373728 weight_movement_bytes=25165824 total_nvlink_bytes=344539552 weight_to_token_ratio=0.078797 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405641144e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158900224 dispatch_meta_bytes=229016 combine_payload_bytes=158900224 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159129240 ingress_bytes=184066048 link_bytes=184066048 token_bytes=318029464 weight_movement_bytes=25165824 total_nvlink_bytes=343195288 weight_to_token_ratio=0.079130 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400529949e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158818304 dispatch_meta_bytes=227640 combine_payload_bytes=158818304 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159045944 ingress_bytes=209149952 link_bytes=209149952 token_bytes=317864248 weight_movement_bytes=50331648 total_nvlink_bytes=368195896 weight_to_token_ratio=0.158343 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591389475e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38844 dup_rows=21116 dup_rows_remote=18518 dedup_ratio=0.322205 router_tokens_per_expert=[520, 575, 514, 557, 518, 533, 477, 495, 510, 519, 507, 504, 527, 474, 541, 490, 535, 515, 498, 480, 511, 510, 491, 505, 503, 516, 501, 475, 521, 525, 539, 507, 481, 494, 481, 528, 514, 477, 506, 517, 488, 567, 541, 495, 541, 502, 502, 497, 500, 521, 495, 476, 512, 513, 534, 512, 509, 520, 517, 498, 579, 536, 506, 516] payload_rows_per_dst=[5584, 5568, 5424, 5576, 5510, 5548, 5582, 5628] dup_rows_per_dst=[2794, 2576, 2666, 2598, 2486, 2718, 2544, 2734] combine_rows_per_src=[5584, 5568, 5424, 5576, 5510, 5548, 5582, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38946 dup_rows=21036 dup_rows_remote=18374 dedup_ratio=0.320984 router_tokens_per_expert=[498, 510, 498, 534, 565, 499, 524, 507, 545, 541, 491, 504, 532, 526, 468, 500, 508, 509, 556, 510, 514, 479, 506, 529, 475, 509, 511, 508, 511, 497, 508, 522, 492, 499, 529, 492, 508, 480, 475, 529, 520, 492, 556, 481, 536, 508, 504, 521, 529, 503, 520, 496, 552, 489, 519, 500, 503, 469, 492, 530, 548, 529, 524, 549] payload_rows_per_dst=[5624, 5588, 5588, 5528, 5456, 5632, 5554, 5530] dup_rows_per_dst=[2646, 2626, 2634, 2554, 2552, 2604, 2662, 2758] combine_rows_per_src=[5624, 5588, 5588, 5528, 5456, 5632, 5554, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44608 payload_rows_remote=39036 dup_rows=20928 dup_rows_remote=18436 dedup_ratio=0.319336 router_tokens_per_expert=[490, 539, 545, 510, 518, 487, 497, 516, 525, 510, 506, 504, 445, 520, 534, 487, 482, 505, 497, 497, 565, 510, 537, 538, 485, 514, 551, 517, 497, 519, 494, 527, 480, 527, 492, 496, 503, 504, 500, 537, 567, 537, 485, 517, 553, 549, 491, 517, 466, 501, 530, 570, 482, 535, 505, 524, 525, 460, 516, 511, 497, 505, 526, 492] payload_rows_per_dst=[5440, 5370, 5638, 5578, 5630, 5796, 5584, 5572] dup_rows_per_dst=[2450, 2396, 2624, 2630, 2762, 2932, 2642, 2492] combine_rows_per_src=[5440, 5370, 5638, 5578, 5630, 5796, 5584, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44366 payload_rows_remote=38848 dup_rows=21170 dup_rows_remote=18542 dedup_ratio=0.323029 router_tokens_per_expert=[541, 572, 509, 487, 531, 502, 508, 497, 492, 528, 482, 500, 509, 535, 518, 509, 490, 506, 511, 509, 507, 483, 519, 521, 532, 506, 522, 491, 558, 519, 517, 532, 502, 508, 536, 549, 487, 532, 501, 518, 538, 567, 487, 531, 482, 531, 506, 487, 516, 509, 481, 525, 529, 504, 519, 473, 509, 494, 498, 472, 481, 502, 548, 503] payload_rows_per_dst=[5624, 5518, 5454, 5614, 5590, 5546, 5530, 5490] dup_rows_per_dst=[2670, 2628, 2638, 2740, 2676, 2712, 2582, 2524] combine_rows_per_src=[5624, 5518, 5454, 5614, 5590, 5546, 5530, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44642 payload_rows_remote=39112 dup_rows=20894 dup_rows_remote=18322 dedup_ratio=0.318817 router_tokens_per_expert=[515, 497, 512, 477, 496, 528, 495, 493, 529, 516, 488, 515, 523, 493, 513, 487, 502, 529, 503, 483, 519, 493, 532, 511, 519, 508, 516, 502, 499, 517, 533, 543, 512, 529, 493, 508, 537, 494, 530, 525, 524, 511, 486, 537, 504, 504, 509, 476, 478, 529, 513, 479, 534, 520, 549, 548, 498, 514, 541, 507, 512, 522, 509, 550] payload_rows_per_dst=[5486, 5598, 5538, 5614, 5638, 5530, 5626, 5612] dup_rows_per_dst=[2540, 2530, 2606, 2660, 2618, 2572, 2674, 2694] combine_rows_per_src=[5486, 5598, 5538, 5614, 5638, 5530, 5626, 5612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44654 payload_rows_remote=39084 dup_rows=20882 dup_rows_remote=18356 dedup_ratio=0.318634 router_tokens_per_expert=[515, 559, 520, 540, 518, 508, 497, 537, 542, 538, 535, 497, 510, 510, 498, 502, 522, 455, 550, 507, 508, 474, 498, 534, 508, 523, 510, 497, 506, 513, 530, 513, 504, 527, 526, 499, 515, 510, 496, 470, 522, 524, 501, 516, 476, 504, 539, 545, 561, 520, 523, 501, 507, 467, 506, 515, 489, 450, 493, 509, 511, 543, 518, 507] payload_rows_per_dst=[5666, 5654, 5570, 5520, 5524, 5614, 5576, 5530] dup_rows_per_dst=[2722, 2610, 2526, 2680, 2570, 2640, 2624, 2510] combine_rows_per_src=[5666, 5654, 5570, 5520, 5524, 5614, 5576, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44324 payload_rows_remote=38730 dup_rows=21212 dup_rows_remote=18542 dedup_ratio=0.323669 router_tokens_per_expert=[494, 503, 526, 523, 479, 488, 518, 488, 503, 490, 492, 497, 537, 493, 528, 539, 528, 526, 478, 506, 471, 499, 524, 496, 533, 498, 496, 521, 497, 517, 527, 518, 498, 520, 496, 523, 542, 501, 524, 528, 524, 532, 501, 541, 533, 512, 530, 511, 496, 501, 474, 507, 536, 514, 529, 508, 537, 512, 532, 541, 512, 521, 483, 516] payload_rows_per_dst=[5432, 5492, 5504, 5588, 5594, 5640, 5524, 5550] dup_rows_per_dst=[2606, 2666, 2552, 2626, 2670, 2728, 2606, 2758] combine_rows_per_src=[5432, 5492, 5504, 5588, 5594, 5640, 5524, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=229448 combine_payload_bytes=159105024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159334472 ingress_bytes=184270848 link_bytes=184270848 token_bytes=318439496 weight_movement_bytes=25165824 total_nvlink_bytes=343605320 weight_to_token_ratio=0.079029 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402088240e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229280 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159752096 ingress_bytes=184688640 link_bytes=184688640 token_bytes=319274912 weight_movement_bytes=25165824 total_nvlink_bytes=344440736 weight_to_token_ratio=0.078822 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405267154e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44360 payload_rows_remote=38860 dup_rows=21176 dup_rows_remote=18556 dedup_ratio=0.323120 router_tokens_per_expert=[510, 506, 485, 503, 487, 517, 532, 513, 533, 526, 525, 512, 499, 535, 536, 522, 517, 475, 477, 511, 484, 471, 488, 509, 516, 509, 512, 545, 495, 490, 445, 551, 534, 546, 509, 531, 495, 496, 491, 519, 513, 485, 518, 545, 543, 529, 512, 508, 559, 506, 502, 508, 485, 517, 486, 498, 479, 527, 556, 551, 529, 519, 506, 530] payload_rows_per_dst=[5500, 5706, 5688, 5542, 5562, 5300, 5526, 5536] dup_rows_per_dst=[2620, 2770, 2948, 2640, 2680, 2234, 2602, 2682] combine_rows_per_src=[5500, 5706, 5688, 5542, 5562, 5300, 5526, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159891456 dispatch_meta_bytes=229888 combine_payload_bytes=159891456 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=160121344 ingress_bytes=260554752 link_bytes=260554752 token_bytes=320012800 weight_movement_bytes=100663296 total_nvlink_bytes=420676096 weight_to_token_ratio=0.314560 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.982520608e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159121408 dispatch_meta_bytes=229560 combine_payload_bytes=159121408 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159350968 ingress_bytes=209453056 link_bytes=209453056 token_bytes=318472376 weight_movement_bytes=50331648 total_nvlink_bytes=368804024 weight_to_token_ratio=0.158041 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593695747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160202752 dispatch_meta_bytes=229736 combine_payload_bytes=160202752 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160432488 ingress_bytes=210534400 link_bytes=210534400 token_bytes=320635240 weight_movement_bytes=50331648 total_nvlink_bytes=370966888 weight_to_token_ratio=0.156975 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601923525e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160088064 dispatch_meta_bytes=229760 combine_payload_bytes=160088064 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160317824 ingress_bytes=185253888 link_bytes=185253888 token_bytes=320405888 weight_movement_bytes=25165824 total_nvlink_bytes=345571712 weight_to_token_ratio=0.078544 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409568039e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158638080 dispatch_meta_bytes=229088 combine_payload_bytes=158638080 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158867168 ingress_bytes=208969728 link_bytes=208969728 token_bytes=317505248 weight_movement_bytes=50331648 total_nvlink_bytes=367836896 weight_to_token_ratio=0.158522 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590018179e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159170560 dispatch_meta_bytes=229664 combine_payload_bytes=159170560 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159400224 ingress_bytes=209502208 link_bytes=209502208 token_bytes=318570784 weight_movement_bytes=50331648 total_nvlink_bytes=368902432 weight_to_token_ratio=0.157992 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594069736e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44574 payload_rows_remote=39038 dup_rows=20962 dup_rows_remote=18420 dedup_ratio=0.319855 router_tokens_per_expert=[525, 541, 513, 496, 503, 521, 542, 468, 505, 527, 491, 501, 505, 483, 509, 499, 564, 467, 507, 524, 516, 543, 516, 497, 499, 514, 499, 458, 525, 521, 515, 508, 526, 460, 526, 536, 507, 493, 540, 511, 513, 532, 514, 525, 506, 529, 490, 501, 500, 503, 532, 529, 502, 519, 477, 542, 510, 489, 531, 544, 525, 552, 499, 503] payload_rows_per_dst=[5666, 5426, 5562, 5536, 5586, 5594, 5608, 5596] dup_rows_per_dst=[2552, 2614, 2706, 2542, 2612, 2626, 2600, 2710] combine_rows_per_src=[5666, 5426, 5562, 5536, 5586, 5594, 5608, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38808 dup_rows=21100 dup_rows_remote=18400 dedup_ratio=0.321960 router_tokens_per_expert=[541, 504, 516, 500, 519, 524, 483, 480, 497, 503, 525, 519, 507, 471, 512, 561, 489, 512, 525, 462, 511, 506, 563, 491, 508, 531, 521, 532, 506, 513, 495, 489, 444, 498, 502, 517, 515, 551, 534, 536, 528, 516, 527, 478, 521, 507, 502, 493, 543, 515, 512, 528, 496, 543, 521, 506, 526, 486, 515, 552, 498, 492, 530, 520] payload_rows_per_dst=[5482, 5544, 5522, 5538, 5602, 5496, 5628, 5624] dup_rows_per_dst=[2652, 2646, 2596, 2652, 2592, 2648, 2700, 2614] combine_rows_per_src=[5482, 5544, 5522, 5538, 5602, 5496, 5628, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44572 payload_rows_remote=39074 dup_rows=20964 dup_rows_remote=18448 dedup_ratio=0.319885 router_tokens_per_expert=[510, 521, 496, 559, 510, 492, 534, 531, 513, 541, 513, 511, 528, 505, 512, 459, 530, 475, 521, 490, 505, 581, 525, 534, 509, 506, 544, 534, 518, 516, 533, 461, 501, 473, 506, 521, 544, 496, 491, 490, 502, 489, 553, 534, 501, 496, 535, 495, 546, 492, 554, 500, 509, 532, 527, 519, 510, 503, 487, 507, 486, 469, 525, 458] payload_rows_per_dst=[5622, 5574, 5584, 5630, 5508, 5462, 5694, 5498] dup_rows_per_dst=[2684, 2590, 2738, 2612, 2536, 2616, 2672, 2516] combine_rows_per_src=[5622, 5574, 5584, 5630, 5508, 5462, 5694, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44504 payload_rows_remote=38874 dup_rows=21032 dup_rows_remote=18268 dedup_ratio=0.320923 router_tokens_per_expert=[511, 506, 478, 510, 485, 530, 524, 498, 511, 541, 558, 507, 552, 493, 515, 520, 524, 505, 525, 474, 561, 515, 480, 527, 523, 489, 564, 517, 518, 524, 478, 527, 491, 498, 516, 519, 525, 528, 512, 525, 513, 499, 509, 467, 514, 502, 519, 511, 547, 477, 519, 512, 513, 505, 498, 502, 502, 529, 505, 512, 511, 515, 527, 456] payload_rows_per_dst=[5540, 5630, 5544, 5600, 5644, 5530, 5534, 5482] dup_rows_per_dst=[2544, 2764, 2678, 2680, 2584, 2538, 2612, 2632] combine_rows_per_src=[5540, 5630, 5544, 5600, 5644, 5530, 5534, 5482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44598 payload_rows_remote=39104 dup_rows=20938 dup_rows_remote=18324 dedup_ratio=0.319489 router_tokens_per_expert=[546, 503, 503, 514, 492, 517, 512, 544, 551, 488, 531, 507, 492, 507, 552, 528, 565, 510, 499, 480, 492, 485, 561, 516, 486, 500, 496, 491, 498, 517, 532, 508, 488, 531, 495, 504, 507, 527, 488, 532, 507, 502, 507, 513, 463, 510, 523, 529, 523, 511, 504, 528, 544, 533, 523, 497, 516, 495, 519, 501, 493, 488, 514, 530] payload_rows_per_dst=[5650, 5624, 5662, 5446, 5546, 5494, 5646, 5530] dup_rows_per_dst=[2612, 2688, 2554, 2610, 2598, 2614, 2680, 2582] combine_rows_per_src=[5650, 5624, 5662, 5446, 5546, 5494, 5646, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44636 payload_rows_remote=39168 dup_rows=20900 dup_rows_remote=18310 dedup_ratio=0.318909 router_tokens_per_expert=[490, 503, 529, 473, 535, 493, 509, 526, 495, 513, 527, 500, 504, 496, 498, 540, 519, 465, 513, 478, 484, 492, 546, 532, 528, 493, 507, 512, 544, 550, 530, 470, 511, 528, 512, 513, 531, 514, 519, 521, 525, 518, 546, 505, 540, 508, 521, 504, 492, 534, 500, 488, 494, 542, 538, 493, 525, 468, 459, 555, 519, 530, 510, 511] payload_rows_per_dst=[5608, 5560, 5468, 5558, 5634, 5654, 5582, 5572] dup_rows_per_dst=[2508, 2586, 2590, 2710, 2664, 2680, 2580, 2582] combine_rows_per_src=[5608, 5560, 5468, 5558, 5634, 5654, 5582, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44378 payload_rows_remote=38830 dup_rows=21158 dup_rows_remote=18542 dedup_ratio=0.322845 router_tokens_per_expert=[508, 498, 520, 495, 497, 526, 498, 502, 521, 545, 532, 506, 513, 523, 482, 511, 517, 511, 479, 494, 565, 500, 504, 531, 536, 490, 480, 520, 524, 544, 538, 494, 501, 531, 503, 510, 538, 529, 503, 467, 510, 514, 498, 498, 531, 539, 511, 490, 520, 518, 532, 481, 521, 505, 497, 523, 534, 511, 524, 498, 525, 522, 487, 493] payload_rows_per_dst=[5452, 5604, 5638, 5544, 5548, 5506, 5492, 5594] dup_rows_per_dst=[2636, 2662, 2564, 2708, 2616, 2676, 2702, 2594] combine_rows_per_src=[5452, 5604, 5638, 5544, 5548, 5506, 5492, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38800 dup_rows=21082 dup_rows_remote=18358 dedup_ratio=0.321686 router_tokens_per_expert=[498, 513, 535, 480, 510, 506, 504, 518, 509, 529, 485, 483, 493, 493, 527, 514, 528, 475, 495, 544, 494, 514, 530, 515, 485, 518, 509, 498, 496, 521, 508, 487, 535, 530, 493, 472, 522, 527, 525, 520, 507, 524, 540, 495, 492, 509, 528, 515, 484, 540, 525, 504, 518, 541, 544, 519, 527, 488, 524, 492, 513, 558, 541, 502] payload_rows_per_dst=[5654, 5560, 5546, 5514, 5600, 5580, 5446, 5554] dup_rows_per_dst=[2724, 2572, 2730, 2694, 2566, 2636, 2424, 2736] combine_rows_per_src=[5654, 5560, 5546, 5514, 5600, 5580, 5446, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159899648 dispatch_meta_bytes=229832 combine_payload_bytes=159899648 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160129480 ingress_bytes=185065472 link_bytes=185065472 token_bytes=320029128 weight_movement_bytes=25165824 total_nvlink_bytes=345194952 weight_to_token_ratio=0.078636 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408134411e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158957568 dispatch_meta_bytes=228832 combine_payload_bytes=158957568 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159186400 ingress_bytes=259620864 link_bytes=259620864 token_bytes=318143968 weight_movement_bytes=100663296 total_nvlink_bytes=418807264 weight_to_token_ratio=0.316408 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975414800e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160047104 dispatch_meta_bytes=230088 combine_payload_bytes=160047104 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160277192 ingress_bytes=185212928 link_bytes=185212928 token_bytes=320324296 weight_movement_bytes=25165824 total_nvlink_bytes=345490120 weight_to_token_ratio=0.078564 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409256380e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159227904 dispatch_meta_bytes=228568 combine_payload_bytes=159227904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159456472 ingress_bytes=184393728 link_bytes=184393728 token_bytes=318684376 weight_movement_bytes=25165824 total_nvlink_bytes=343850200 weight_to_token_ratio=0.078968 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403023215e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160432128 dispatch_meta_bytes=229912 combine_payload_bytes=160432128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160662040 ingress_bytes=185597952 link_bytes=185597952 token_bytes=321094168 weight_movement_bytes=25165824 total_nvlink_bytes=346259992 weight_to_token_ratio=0.078375 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.412185968e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160169984 dispatch_meta_bytes=229712 combine_payload_bytes=160169984 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160399696 ingress_bytes=235667456 link_bytes=235667456 token_bytes=320569680 weight_movement_bytes=75497472 total_nvlink_bytes=396067152 weight_to_token_ratio=0.235510 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.793157041e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159047680 dispatch_meta_bytes=229488 combine_payload_bytes=159047680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159277168 ingress_bytes=184213504 link_bytes=184213504 token_bytes=318324848 weight_movement_bytes=25165824 total_nvlink_bytes=343490672 weight_to_token_ratio=0.079057 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401651918e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158924800 dispatch_meta_bytes=228632 combine_payload_bytes=158924800 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159153432 ingress_bytes=184090624 link_bytes=184090624 token_bytes=318078232 weight_movement_bytes=25165824 total_nvlink_bytes=343244056 weight_to_token_ratio=0.079118 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400716944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38946 dup_rows=21036 dup_rows_remote=18352 dedup_ratio=0.320984 router_tokens_per_expert=[532, 487, 534, 543, 509, 501, 488, 471, 511, 499, 533, 527, 524, 499, 526, 483, 476, 507, 488, 515, 488, 550, 512, 519, 506, 515, 514, 509, 518, 500, 524, 533, 497, 481, 506, 523, 531, 467, 481, 497, 529, 515, 536, 523, 511, 512, 481, 508, 531, 461, 516, 499, 541, 525, 511, 516, 553, 523, 570, 534, 500, 501, 508, 540] payload_rows_per_dst=[5544, 5616, 5524, 5554, 5474, 5556, 5608, 5624] dup_rows_per_dst=[2586, 2588, 2586, 2684, 2492, 2674, 2592, 2834] combine_rows_per_src=[5544, 5616, 5524, 5554, 5474, 5556, 5608, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44572 payload_rows_remote=39072 dup_rows=20964 dup_rows_remote=18298 dedup_ratio=0.319885 router_tokens_per_expert=[539, 537, 538, 536, 480, 526, 531, 493, 516, 477, 509, 505, 518, 541, 508, 468, 517, 539, 498, 510, 534, 498, 512, 487, 514, 499, 460, 509, 527, 492, 544, 521, 547, 493, 505, 495, 537, 532, 536, 505, 498, 505, 535, 486, 538, 498, 506, 511, 476, 497, 537, 522, 513, 509, 511, 518, 515, 517, 475, 525, 484, 526, 507, 526] payload_rows_per_dst=[5670, 5558, 5472, 5602, 5668, 5546, 5500, 5556] dup_rows_per_dst=[2690, 2526, 2718, 2530, 2632, 2608, 2666, 2594] combine_rows_per_src=[5670, 5558, 5472, 5602, 5668, 5546, 5500, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44654 payload_rows_remote=39048 dup_rows=20882 dup_rows_remote=18210 dedup_ratio=0.318634 router_tokens_per_expert=[521, 516, 500, 501, 504, 526, 526, 513, 514, 504, 507, 497, 511, 519, 526, 540, 478, 500, 522, 495, 472, 540, 525, 496, 508, 501, 483, 507, 527, 472, 521, 511, 532, 485, 479, 536, 515, 551, 516, 515, 493, 532, 499, 516, 537, 528, 520, 519, 508, 506, 543, 545, 495, 479, 519, 494, 516, 520, 535, 509, 508, 524, 502, 509] payload_rows_per_dst=[5542, 5570, 5490, 5498, 5662, 5664, 5622, 5606] dup_rows_per_dst=[2538, 2582, 2566, 2562, 2596, 2684, 2682, 2672] combine_rows_per_src=[5542, 5570, 5490, 5498, 5662, 5664, 5622, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44334 payload_rows_remote=38878 dup_rows=21202 dup_rows_remote=18488 dedup_ratio=0.323517 router_tokens_per_expert=[540, 534, 463, 531, 482, 527, 519, 530, 521, 521, 513, 519, 502, 511, 489, 509, 489, 512, 504, 522, 523, 491, 498, 526, 519, 522, 470, 508, 512, 485, 488, 509, 543, 542, 474, 500, 522, 531, 510, 518, 521, 499, 522, 505, 521, 518, 529, 500, 494, 513, 510, 482, 487, 507, 522, 508, 499, 528, 565, 554, 552, 537, 511, 455] payload_rows_per_dst=[5600, 5456, 5524, 5536, 5602, 5536, 5502, 5578] dup_rows_per_dst=[2652, 2714, 2606, 2490, 2678, 2694, 2544, 2824] combine_rows_per_src=[5600, 5456, 5524, 5536, 5602, 5536, 5502, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44318 payload_rows_remote=38854 dup_rows=21218 dup_rows_remote=18618 dedup_ratio=0.323761 router_tokens_per_expert=[517, 482, 529, 515, 501, 541, 498, 546, 515, 505, 477, 489, 534, 503, 535, 550, 500, 515, 507, 529, 469, 527, 491, 494, 527, 488, 504, 526, 533, 532, 521, 518, 498, 538, 511, 459, 510, 480, 549, 468, 495, 500, 523, 533, 516, 528, 495, 523, 498, 564, 501, 488, 543, 511, 535, 504, 505, 498, 539, 524, 521, 498, 505, 490] payload_rows_per_dst=[5598, 5618, 5464, 5622, 5476, 5496, 5524, 5520] dup_rows_per_dst=[2660, 2598, 2600, 2676, 2550, 2730, 2764, 2640] combine_rows_per_src=[5598, 5618, 5464, 5622, 5476, 5496, 5524, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44728 payload_rows_remote=39122 dup_rows=20808 dup_rows_remote=18262 dedup_ratio=0.317505 router_tokens_per_expert=[526, 548, 529, 521, 505, 479, 516, 509, 530, 485, 526, 513, 467, 530, 510, 563, 515, 506, 493, 491, 495, 516, 541, 508, 505, 522, 498, 511, 505, 474, 508, 505, 504, 513, 492, 564, 487, 533, 532, 522, 500, 490, 543, 524, 497, 502, 493, 527, 460, 510, 515, 511, 512, 530, 472, 537, 537, 480, 551, 517, 541, 515, 507, 500] payload_rows_per_dst=[5614, 5628, 5592, 5482, 5604, 5606, 5540, 5662] dup_rows_per_dst=[2652, 2620, 2538, 2574, 2690, 2546, 2554, 2634] combine_rows_per_src=[5614, 5628, 5592, 5482, 5604, 5606, 5540, 5662] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229192 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159752008 ingress_bytes=184688640 link_bytes=184688640 token_bytes=319274824 weight_movement_bytes=25165824 total_nvlink_bytes=344440648 weight_to_token_ratio=0.078822 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405267154e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44770 payload_rows_remote=39126 dup_rows=20766 dup_rows_remote=18116 dedup_ratio=0.316864 router_tokens_per_expert=[524, 538, 535, 503, 476, 555, 534, 509, 504, 514, 499, 521, 537, 512, 503, 493, 536, 516, 498, 483, 498, 485, 533, 484, 503, 498, 499, 530, 526, 506, 523, 517, 513, 529, 533, 486, 512, 518, 489, 567, 497, 505, 508, 501, 539, 548, 482, 518, 500, 497, 533, 492, 511, 496, 521, 535, 518, 504, 506, 536, 452, 518, 492, 520] payload_rows_per_dst=[5624, 5612, 5554, 5560, 5644, 5618, 5634, 5524] dup_rows_per_dst=[2724, 2554, 2512, 2644, 2650, 2578, 2536, 2568] combine_rows_per_src=[5624, 5612, 5554, 5560, 5644, 5618, 5634, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160038912 dispatch_meta_bytes=229480 combine_payload_bytes=160038912 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160268392 ingress_bytes=185204736 link_bytes=185204736 token_bytes=320307304 weight_movement_bytes=25165824 total_nvlink_bytes=345473128 weight_to_token_ratio=0.078568 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409194049e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38992 dup_rows=21050 dup_rows_remote=18442 dedup_ratio=0.321198 router_tokens_per_expert=[495, 490, 469, 522, 537, 518, 515, 505, 493, 499, 520, 474, 516, 506, 516, 574, 496, 515, 482, 492, 511, 498, 521, 536, 544, 527, 498, 531, 497, 515, 525, 470, 502, 508, 519, 504, 492, 500, 500, 505, 500, 520, 532, 531, 537, 471, 539, 507, 501, 504, 497, 499, 529, 537, 555, 500, 509, 494, 532, 531, 544, 512, 508, 542] payload_rows_per_dst=[5494, 5606, 5754, 5676, 5636, 5442, 5580, 5298] dup_rows_per_dst=[2608, 2590, 2900, 2798, 2612, 2572, 2664, 2306] combine_rows_per_src=[5494, 5606, 5754, 5676, 5636, 5442, 5580, 5298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159940608 dispatch_meta_bytes=229032 combine_payload_bytes=159940608 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160169640 ingress_bytes=235438080 link_bytes=235438080 token_bytes=320110248 weight_movement_bytes=75497472 total_nvlink_bytes=395607720 weight_to_token_ratio=0.235848 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791411755e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=229464 combine_payload_bytes=159244288 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159473752 ingress_bytes=234741760 link_bytes=234741760 token_bytes=318718040 weight_movement_bytes=75497472 total_nvlink_bytes=394215512 weight_to_token_ratio=0.236879 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786113564e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160243712 dispatch_meta_bytes=229536 combine_payload_bytes=160243712 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160473248 ingress_bytes=210575360 link_bytes=210575360 token_bytes=320716960 weight_movement_bytes=50331648 total_nvlink_bytes=371048608 weight_to_token_ratio=0.156935 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602235183e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159145984 dispatch_meta_bytes=229888 combine_payload_bytes=159145984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159375872 ingress_bytes=184311808 link_bytes=184311808 token_bytes=318521856 weight_movement_bytes=25165824 total_nvlink_bytes=343687680 weight_to_token_ratio=0.079008 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402399898e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160260096 dispatch_meta_bytes=228968 combine_payload_bytes=160260096 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160489064 ingress_bytes=235757568 link_bytes=235757568 token_bytes=320749160 weight_movement_bytes=75497472 total_nvlink_bytes=396246632 weight_to_token_ratio=0.235379 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.793842690e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159711232 dispatch_meta_bytes=229736 combine_payload_bytes=159711232 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159940968 ingress_bytes=210042880 link_bytes=210042880 token_bytes=319652200 weight_movement_bytes=50331648 total_nvlink_bytes=369983848 weight_to_token_ratio=0.157458 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598183626e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44400 payload_rows_remote=38810 dup_rows=21136 dup_rows_remote=18436 dedup_ratio=0.322510 router_tokens_per_expert=[543, 493, 511, 490, 520, 484, 545, 484, 510, 498, 519, 491, 521, 474, 506, 511, 495, 500, 527, 469, 476, 522, 524, 518, 504, 526, 514, 484, 532, 546, 533, 506, 524, 486, 527, 511, 474, 550, 511, 516, 525, 504, 528, 527, 504, 493, 498, 503, 523, 506, 542, 558, 521, 505, 526, 513, 519, 542, 502, 490, 529, 511, 505, 519] payload_rows_per_dst=[5570, 5508, 5498, 5590, 5562, 5508, 5630, 5534] dup_rows_per_dst=[2570, 2552, 2564, 2700, 2636, 2656, 2758, 2700] combine_rows_per_src=[5570, 5508, 5498, 5590, 5562, 5508, 5630, 5534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=39044 dup_rows=20982 dup_rows_remote=18492 dedup_ratio=0.320160 router_tokens_per_expert=[523, 518, 538, 489, 534, 515, 521, 532, 516, 505, 485, 540, 522, 558, 494, 524, 527, 549, 527, 481, 522, 486, 509, 522, 488, 502, 530, 506, 513, 469, 521, 526, 499, 500, 509, 545, 510, 492, 559, 537, 500, 482, 547, 512, 499, 508, 457, 497, 481, 524, 485, 512, 495, 514, 474, 515, 528, 500, 506, 491, 495, 525, 493, 585] payload_rows_per_dst=[5664, 5620, 5644, 5516, 5606, 5456, 5510, 5538] dup_rows_per_dst=[2676, 2668, 2602, 2594, 2696, 2548, 2490, 2708] combine_rows_per_src=[5664, 5620, 5644, 5516, 5606, 5456, 5510, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44266 payload_rows_remote=38644 dup_rows=21270 dup_rows_remote=18594 dedup_ratio=0.324554 router_tokens_per_expert=[537, 485, 482, 498, 526, 526, 490, 536, 508, 579, 484, 539, 543, 521, 514, 516, 503, 504, 498, 488, 509, 516, 510, 534, 500, 525, 541, 524, 505, 549, 488, 488, 504, 511, 513, 510, 518, 497, 526, 538, 503, 469, 513, 516, 483, 546, 522, 513, 478, 491, 486, 487, 519, 500, 526, 513, 522, 530, 500, 527, 530, 471, 520, 520] payload_rows_per_dst=[5410, 5530, 5520, 5548, 5522, 5666, 5448, 5622] dup_rows_per_dst=[2492, 2560, 2604, 2682, 2624, 3040, 2592, 2676] combine_rows_per_src=[5410, 5530, 5520, 5548, 5522, 5666, 5448, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=39090 dup_rows=20910 dup_rows_remote=18338 dedup_ratio=0.319061 router_tokens_per_expert=[546, 488, 488, 509, 514, 509, 532, 512, 488, 510, 531, 505, 518, 522, 522, 474, 460, 540, 506, 577, 548, 534, 490, 546, 511, 539, 517, 517, 498, 496, 499, 521, 484, 520, 513, 512, 500, 495, 486, 531, 516, 492, 505, 511, 507, 492, 521, 510, 532, 515, 512, 539, 523, 504, 517, 541, 497, 520, 473, 495, 557, 514, 495, 472] payload_rows_per_dst=[5544, 5534, 5692, 5630, 5420, 5536, 5698, 5572] dup_rows_per_dst=[2652, 2606, 2710, 2566, 2662, 2572, 2668, 2474] combine_rows_per_src=[5544, 5534, 5692, 5630, 5420, 5536, 5698, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=39040 dup_rows=20990 dup_rows_remote=18300 dedup_ratio=0.320282 router_tokens_per_expert=[528, 498, 538, 548, 524, 513, 540, 531, 549, 516, 513, 514, 501, 480, 481, 544, 496, 497, 520, 513, 511, 513, 521, 485, 518, 506, 491, 527, 480, 499, 475, 504, 542, 459, 519, 531, 481, 507, 559, 511, 569, 497, 523, 518, 490, 506, 480, 498, 519, 512, 501, 500, 509, 505, 538, 499, 479, 500, 518, 535, 472, 524, 542, 551] payload_rows_per_dst=[5652, 5506, 5494, 5478, 5610, 5586, 5612, 5608] dup_rows_per_dst=[2788, 2690, 2618, 2522, 2608, 2576, 2554, 2634] combine_rows_per_src=[5652, 5506, 5494, 5478, 5610, 5586, 5612, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158965760 dispatch_meta_bytes=228984 combine_payload_bytes=158965760 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159194744 ingress_bytes=184131584 link_bytes=184131584 token_bytes=318160504 weight_movement_bytes=25165824 total_nvlink_bytes=343326328 weight_to_token_ratio=0.079098 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401028602e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=39048 dup_rows=21072 dup_rows_remote=18538 dedup_ratio=0.321533 router_tokens_per_expert=[501, 513, 526, 500, 507, 484, 535, 520, 509, 523, 508, 528, 503, 523, 550, 539, 484, 474, 515, 500, 488, 485, 503, 526, 560, 525, 506, 515, 527, 522, 525, 481, 502, 462, 507, 493, 513, 485, 522, 549, 506, 513, 502, 480, 508, 509, 508, 489, 483, 510, 541, 554, 506, 523, 546, 526, 506, 528, 526, 545, 516, 506, 487, 512] payload_rows_per_dst=[5546, 5628, 5416, 5656, 5534, 5460, 5666, 5558] dup_rows_per_dst=[2626, 2738, 2534, 2666, 2532, 2570, 2712, 2694] combine_rows_per_src=[5546, 5628, 5416, 5656, 5534, 5460, 5666, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44258 payload_rows_remote=38702 dup_rows=21278 dup_rows_remote=18574 dedup_ratio=0.324677 router_tokens_per_expert=[479, 524, 467, 511, 548, 520, 535, 490, 489, 480, 535, 497, 502, 523, 547, 495, 543, 544, 514, 508, 526, 531, 511, 510, 512, 506, 532, 518, 543, 494, 526, 517, 524, 513, 514, 509, 503, 511, 515, 541, 468, 503, 539, 522, 500, 505, 503, 508, 477, 515, 511, 452, 479, 557, 507, 566, 524, 519, 500, 499, 475, 463, 544, 525] payload_rows_per_dst=[5524, 5558, 5580, 5528, 5556, 5476, 5522, 5514] dup_rows_per_dst=[2624, 2578, 2794, 2768, 2704, 2620, 2606, 2584] combine_rows_per_src=[5524, 5558, 5580, 5528, 5556, 5476, 5522, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159924224 dispatch_meta_bytes=230144 combine_payload_bytes=159924224 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160154368 ingress_bytes=210255872 link_bytes=210255872 token_bytes=320078592 weight_movement_bytes=50331648 total_nvlink_bytes=370410240 weight_to_token_ratio=0.157248 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599804249e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44488 payload_rows_remote=39006 dup_rows=21048 dup_rows_remote=18532 dedup_ratio=0.321167 router_tokens_per_expert=[493, 451, 510, 490, 518, 499, 508, 530, 519, 557, 541, 481, 496, 509, 522, 490, 534, 484, 535, 500, 509, 526, 558, 535, 543, 527, 478, 508, 513, 500, 491, 532, 525, 538, 507, 526, 533, 484, 497, 505, 526, 525, 526, 471, 498, 527, 492, 518, 517, 496, 505, 529, 536, 502, 509, 526, 489, 492, 493, 525, 500, 514, 523, 527] payload_rows_per_dst=[5482, 5542, 5712, 5596, 5564, 5540, 5504, 5548] dup_rows_per_dst=[2516, 2688, 2862, 2588, 2666, 2626, 2524, 2578] combine_rows_per_src=[5482, 5542, 5712, 5596, 5564, 5540, 5504, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158285824 dispatch_meta_bytes=228952 combine_payload_bytes=158285824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158514776 ingress_bytes=183451648 link_bytes=183451648 token_bytes=316800600 weight_movement_bytes=25165824 total_nvlink_bytes=341966424 weight_to_token_ratio=0.079437 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.395855075e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160112640 dispatch_meta_bytes=229712 combine_payload_bytes=160112640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160342352 ingress_bytes=185278464 link_bytes=185278464 token_bytes=320454992 weight_movement_bytes=25165824 total_nvlink_bytes=345620816 weight_to_token_ratio=0.078532 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409755034e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159907840 dispatch_meta_bytes=229360 combine_payload_bytes=159907840 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160137200 ingress_bytes=210239488 link_bytes=210239488 token_bytes=320045040 weight_movement_bytes=50331648 total_nvlink_bytes=370376688 weight_to_token_ratio=0.157264 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599679585e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159940608 dispatch_meta_bytes=230344 combine_payload_bytes=159940608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160170952 ingress_bytes=185106432 link_bytes=185106432 token_bytes=320111560 weight_movement_bytes=25165824 total_nvlink_bytes=345277384 weight_to_token_ratio=0.078616 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408446069e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158523392 dispatch_meta_bytes=229104 combine_payload_bytes=158523392 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158752496 ingress_bytes=208855040 link_bytes=208855040 token_bytes=317275888 weight_movement_bytes=50331648 total_nvlink_bytes=367607536 weight_to_token_ratio=0.158637 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589145536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159768576 dispatch_meta_bytes=230152 combine_payload_bytes=159768576 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159998728 ingress_bytes=184934400 link_bytes=184934400 token_bytes=319767304 weight_movement_bytes=25165824 total_nvlink_bytes=344933128 weight_to_token_ratio=0.078700 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407137104e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44606 payload_rows_remote=38986 dup_rows=20930 dup_rows_remote=18374 dedup_ratio=0.319366 router_tokens_per_expert=[486, 538, 484, 483, 509, 510, 535, 549, 528, 523, 514, 497, 485, 505, 493, 463, 485, 509, 531, 506, 491, 522, 525, 513, 486, 512, 490, 507, 501, 516, 522, 554, 498, 513, 509, 544, 503, 531, 514, 549, 520, 547, 542, 509, 503, 485, 499, 473, 515, 509, 493, 511, 529, 515, 505, 546, 532, 466, 508, 493, 545, 533, 502, 555] payload_rows_per_dst=[5562, 5488, 5552, 5620, 5646, 5528, 5562, 5648] dup_rows_per_dst=[2626, 2528, 2612, 2556, 2676, 2628, 2684, 2620] combine_rows_per_src=[5562, 5488, 5552, 5620, 5646, 5528, 5562, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44592 payload_rows_remote=39036 dup_rows=20944 dup_rows_remote=18314 dedup_ratio=0.319580 router_tokens_per_expert=[525, 510, 521, 499, 518, 495, 542, 481, 500, 516, 531, 529, 511, 481, 545, 556, 513, 509, 501, 486, 529, 560, 507, 520, 458, 497, 536, 522, 508, 487, 504, 514, 500, 515, 476, 471, 503, 503, 529, 530, 531, 522, 520, 519, 516, 511, 505, 472, 549, 507, 524, 448, 519, 522, 510, 514, 535, 518, 558, 511, 493, 514, 524, 488] payload_rows_per_dst=[5564, 5656, 5574, 5552, 5506, 5564, 5556, 5620] dup_rows_per_dst=[2618, 2682, 2676, 2500, 2548, 2628, 2630, 2662] combine_rows_per_src=[5564, 5656, 5574, 5552, 5506, 5564, 5556, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44540 payload_rows_remote=39016 dup_rows=20996 dup_rows_remote=18394 dedup_ratio=0.320374 router_tokens_per_expert=[545, 530, 516, 536, 509, 531, 501, 517, 512, 516, 498, 468, 516, 493, 490, 523, 519, 499, 505, 517, 549, 513, 490, 472, 524, 482, 529, 518, 505, 542, 500, 488, 505, 516, 486, 510, 529, 508, 522, 514, 523, 453, 534, 526, 562, 464, 514, 487, 528, 485, 535, 531, 539, 511, 524, 537, 483, 479, 502, 510, 531, 522, 535, 510] payload_rows_per_dst=[5660, 5508, 5520, 5552, 5576, 5524, 5650, 5550] dup_rows_per_dst=[2710, 2524, 2608, 2624, 2604, 2602, 2730, 2594] combine_rows_per_src=[5660, 5508, 5520, 5552, 5576, 5524, 5650, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44392 payload_rows_remote=38832 dup_rows=21144 dup_rows_remote=18458 dedup_ratio=0.322632 router_tokens_per_expert=[502, 495, 507, 495, 507, 511, 507, 522, 538, 521, 498, 486, 531, 535, 516, 511, 518, 493, 540, 520, 499, 486, 515, 547, 492, 513, 504, 498, 474, 506, 517, 453, 514, 512, 506, 534, 502, 513, 518, 506, 506, 550, 509, 501, 527, 531, 526, 523, 534, 532, 486, 509, 497, 534, 514, 506, 511, 508, 512, 540, 512, 508, 547, 483] payload_rows_per_dst=[5516, 5584, 5526, 5412, 5566, 5600, 5628, 5560] dup_rows_per_dst=[2576, 2688, 2612, 2502, 2742, 2746, 2592, 2686] combine_rows_per_src=[5516, 5584, 5526, 5412, 5566, 5600, 5628, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159686656 dispatch_meta_bytes=229440 combine_payload_bytes=159686656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159916096 ingress_bytes=184852480 link_bytes=184852480 token_bytes=319602752 weight_movement_bytes=25165824 total_nvlink_bytes=344768576 weight_to_token_ratio=0.078741 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406513788e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38800 dup_rows=21166 dup_rows_remote=18568 dedup_ratio=0.322968 router_tokens_per_expert=[494, 565, 565, 516, 521, 465, 471, 470, 495, 474, 478, 520, 535, 532, 504, 546, 508, 460, 498, 527, 511, 500, 491, 526, 518, 461, 499, 535, 512, 501, 499, 489, 513, 536, 477, 549, 532, 541, 508, 536, 529, 513, 504, 516, 507, 541, 515, 502, 505, 517, 485, 534, 526, 508, 511, 501, 463, 544, 543, 538, 543, 531, 485, 529] payload_rows_per_dst=[5440, 5570, 5460, 5430, 5674, 5620, 5544, 5632] dup_rows_per_dst=[2694, 2598, 2582, 2598, 2710, 2634, 2630, 2720] combine_rows_per_src=[5440, 5570, 5460, 5430, 5674, 5620, 5544, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44420 payload_rows_remote=38940 dup_rows=21116 dup_rows_remote=18484 dedup_ratio=0.322205 router_tokens_per_expert=[519, 478, 522, 516, 478, 517, 507, 492, 492, 544, 481, 511, 514, 466, 507, 529, 519, 525, 544, 505, 508, 514, 548, 510, 538, 530, 524, 517, 503, 513, 496, 524, 482, 507, 499, 537, 485, 485, 526, 535, 522, 479, 534, 555, 498, 496, 507, 515, 523, 509, 472, 524, 538, 543, 519, 526, 506, 519, 475, 570, 501, 491, 492, 507] payload_rows_per_dst=[5518, 5496, 5594, 5630, 5480, 5592, 5586, 5524] dup_rows_per_dst=[2540, 2592, 2752, 2660, 2632, 2620, 2722, 2598] combine_rows_per_src=[5518, 5496, 5594, 5630, 5480, 5592, 5586, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39048 dup_rows=20934 dup_rows_remote=18228 dedup_ratio=0.319427 router_tokens_per_expert=[490, 528, 490, 515, 513, 519, 531, 518, 525, 539, 519, 496, 509, 494, 549, 513, 477, 503, 470, 522, 532, 527, 493, 488, 484, 532, 537, 501, 498, 526, 478, 519, 535, 519, 507, 497, 530, 523, 530, 506, 498, 488, 519, 525, 516, 502, 527, 547, 515, 501, 474, 520, 521, 489, 529, 544, 504, 515, 493, 477, 528, 526, 488, 540] payload_rows_per_dst=[5554, 5662, 5514, 5792, 5378, 5626, 5552, 5524] dup_rows_per_dst=[2706, 2740, 2510, 3006, 2268, 2632, 2506, 2566] combine_rows_per_src=[5554, 5662, 5514, 5792, 5378, 5626, 5552, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159891456 dispatch_meta_bytes=229400 combine_payload_bytes=159891456 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160120856 ingress_bytes=235388928 link_bytes=235388928 token_bytes=320012312 weight_movement_bytes=75497472 total_nvlink_bytes=395509784 weight_to_token_ratio=0.235921 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791037765e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44582 payload_rows_remote=38884 dup_rows=20954 dup_rows_remote=18236 dedup_ratio=0.319733 router_tokens_per_expert=[486, 485, 533, 514, 500, 527, 482, 554, 478, 513, 542, 517, 532, 488, 487, 529, 549, 518, 508, 505, 540, 549, 504, 535, 478, 516, 498, 508, 512, 528, 486, 522, 491, 543, 526, 531, 510, 506, 509, 575, 512, 492, 528, 474, 553, 514, 471, 510, 525, 512, 488, 524, 502, 493, 506, 522, 514, 483, 461, 522, 504, 522, 491, 531] payload_rows_per_dst=[5524, 5594, 5698, 5470, 5630, 5564, 5524, 5578] dup_rows_per_dst=[2638, 2578, 2718, 2626, 2752, 2544, 2620, 2478] combine_rows_per_src=[5524, 5594, 5698, 5470, 5630, 5564, 5524, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159809536 dispatch_meta_bytes=229640 combine_payload_bytes=159809536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160039176 ingress_bytes=184975360 link_bytes=184975360 token_bytes=319848712 weight_movement_bytes=25165824 total_nvlink_bytes=345014536 weight_to_token_ratio=0.078680 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407448762e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159055872 dispatch_meta_bytes=229160 combine_payload_bytes=159055872 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159285032 ingress_bytes=209387520 link_bytes=209387520 token_bytes=318340904 weight_movement_bytes=50331648 total_nvlink_bytes=368672552 weight_to_token_ratio=0.158106 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593197093e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158924800 dispatch_meta_bytes=229472 combine_payload_bytes=158924800 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159154272 ingress_bytes=184090624 link_bytes=184090624 token_bytes=318079072 weight_movement_bytes=25165824 total_nvlink_bytes=343244896 weight_to_token_ratio=0.079118 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400716944e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=229696 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159727936 ingress_bytes=184664064 link_bytes=184664064 token_bytes=319226176 weight_movement_bytes=25165824 total_nvlink_bytes=344392000 weight_to_token_ratio=0.078834 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405080160e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159940608 dispatch_meta_bytes=229104 combine_payload_bytes=159940608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160169712 ingress_bytes=185106432 link_bytes=185106432 token_bytes=320110320 weight_movement_bytes=25165824 total_nvlink_bytes=345276144 weight_to_token_ratio=0.078616 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408446069e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=228480 combine_payload_bytes=159268864 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159497344 ingress_bytes=184434688 link_bytes=184434688 token_bytes=318766208 weight_movement_bytes=25165824 total_nvlink_bytes=343932032 weight_to_token_ratio=0.078948 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403334873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=38994 dup_rows=20946 dup_rows_remote=18280 dedup_ratio=0.319611 router_tokens_per_expert=[540, 528, 526, 519, 537, 534, 552, 484, 527, 533, 484, 572, 533, 502, 500, 518, 515, 479, 517, 511, 505, 495, 510, 499, 503, 504, 519, 552, 548, 514, 482, 509, 511, 511, 509, 457, 508, 499, 487, 490, 505, 511, 557, 506, 551, 520, 507, 515, 516, 508, 474, 502, 512, 511, 498, 497, 508, 505, 496, 555, 484, 506, 497, 504] payload_rows_per_dst=[5672, 5602, 5550, 5596, 5464, 5602, 5522, 5582] dup_rows_per_dst=[2768, 2736, 2512, 2666, 2480, 2742, 2514, 2528] combine_rows_per_src=[5672, 5602, 5550, 5596, 5464, 5602, 5522, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44334 payload_rows_remote=38780 dup_rows=21202 dup_rows_remote=18550 dedup_ratio=0.323517 router_tokens_per_expert=[508, 527, 483, 512, 535, 499, 533, 516, 529, 540, 513, 505, 503, 510, 481, 519, 497, 505, 502, 541, 497, 567, 489, 521, 534, 484, 548, 535, 492, 551, 510, 512, 524, 506, 532, 500, 494, 522, 492, 491, 513, 558, 526, 508, 483, 497, 496, 497, 540, 497, 517, 481, 517, 524, 521, 506, 527, 498, 471, 483, 519, 505, 518, 507] payload_rows_per_dst=[5572, 5572, 5548, 5608, 5462, 5496, 5554, 5522] dup_rows_per_dst=[2654, 2628, 2690, 2724, 2660, 2660, 2652, 2534] combine_rows_per_src=[5572, 5572, 5548, 5608, 5462, 5496, 5554, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159719424 dispatch_meta_bytes=229096 combine_payload_bytes=159719424 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159948520 ingress_bytes=210051072 link_bytes=210051072 token_bytes=319667944 weight_movement_bytes=50331648 total_nvlink_bytes=369999592 weight_to_token_ratio=0.157450 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598245957e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44604 payload_rows_remote=39118 dup_rows=20932 dup_rows_remote=18282 dedup_ratio=0.319397 router_tokens_per_expert=[540, 488, 490, 504, 531, 517, 512, 493, 484, 539, 507, 496, 517, 468, 491, 502, 514, 534, 495, 503, 546, 505, 515, 520, 545, 533, 514, 509, 483, 499, 491, 520, 526, 505, 516, 539, 513, 484, 502, 470, 522, 488, 533, 490, 521, 493, 497, 524, 525, 512, 517, 553, 545, 497, 536, 536, 490, 508, 505, 547, 486, 511, 538, 534] payload_rows_per_dst=[5542, 5526, 5606, 5666, 5506, 5486, 5686, 5586] dup_rows_per_dst=[2608, 2482, 2658, 2522, 2604, 2650, 2756, 2652] combine_rows_per_src=[5542, 5526, 5606, 5666, 5506, 5486, 5686, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38670 dup_rows=21114 dup_rows_remote=18114 dedup_ratio=0.322174 router_tokens_per_expert=[503, 526, 488, 533, 499, 531, 516, 492, 510, 480, 513, 475, 538, 499, 491, 515, 521, 508, 527, 456, 475, 510, 504, 514, 535, 488, 492, 500, 517, 537, 502, 524, 527, 515, 509, 513, 504, 525, 474, 524, 544, 509, 507, 514, 555, 520, 518, 496, 498, 517, 538, 536, 515, 507, 516, 526, 503, 527, 520, 484, 515, 523, 507, 563] payload_rows_per_dst=[5312, 5538, 5378, 5578, 5554, 5666, 5644, 5752] dup_rows_per_dst=[2396, 2504, 2424, 2724, 2628, 2776, 2662, 3000] combine_rows_per_src=[5312, 5538, 5378, 5578, 5554, 5666, 5644, 5752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44596 payload_rows_remote=38970 dup_rows=20940 dup_rows_remote=18378 dedup_ratio=0.319519 router_tokens_per_expert=[551, 503, 560, 504, 508, 543, 506, 521, 505, 509, 537, 510, 503, 497, 515, 518, 544, 513, 507, 520, 504, 475, 540, 512, 518, 511, 504, 526, 497, 533, 522, 504, 508, 512, 480, 489, 508, 513, 516, 483, 514, 529, 525, 522, 504, 534, 526, 533, 484, 522, 526, 527, 501, 464, 533, 536, 494, 524, 496, 492, 473, 501, 484, 495] payload_rows_per_dst=[5682, 5626, 5642, 5550, 5432, 5612, 5568, 5484] dup_rows_per_dst=[2710, 2562, 2588, 2680, 2586, 2762, 2618, 2434] combine_rows_per_src=[5682, 5626, 5642, 5550, 5432, 5612, 5568, 5484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44694 payload_rows_remote=39092 dup_rows=20842 dup_rows_remote=18270 dedup_ratio=0.318024 router_tokens_per_expert=[510, 513, 510, 516, 524, 484, 513, 502, 494, 518, 511, 523, 529, 534, 555, 503, 503, 498, 487, 512, 559, 491, 491, 511, 526, 549, 499, 512, 502, 533, 480, 537, 481, 517, 500, 503, 516, 548, 521, 501, 487, 544, 509, 459, 522, 531, 562, 515, 544, 538, 493, 521, 505, 492, 501, 517, 505, 484, 495, 486, 531, 491, 514, 506] payload_rows_per_dst=[5660, 5630, 5576, 5570, 5602, 5608, 5594, 5454] dup_rows_per_dst=[2484, 2704, 2528, 2706, 2572, 2650, 2628, 2570] combine_rows_per_src=[5660, 5630, 5576, 5570, 5602, 5608, 5594, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44690 payload_rows_remote=39080 dup_rows=20846 dup_rows_remote=18120 dedup_ratio=0.318085 router_tokens_per_expert=[556, 497, 515, 542, 505, 503, 493, 488, 496, 515, 505, 522, 497, 489, 490, 532, 505, 483, 496, 523, 522, 520, 518, 492, 515, 536, 545, 492, 514, 527, 494, 522, 511, 513, 507, 524, 527, 521, 499, 520, 512, 509, 486, 521, 519, 519, 543, 522, 505, 518, 498, 487, 513, 494, 520, 514, 507, 562, 494, 506, 488, 521, 518, 521] payload_rows_per_dst=[5610, 5692, 5588, 5452, 5838, 5442, 5476, 5592] dup_rows_per_dst=[2726, 2878, 2584, 2360, 2940, 2286, 2430, 2642] combine_rows_per_src=[5610, 5692, 5588, 5452, 5838, 5442, 5476, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44334 payload_rows_remote=38706 dup_rows=21202 dup_rows_remote=18408 dedup_ratio=0.323517 router_tokens_per_expert=[477, 489, 531, 478, 543, 542, 517, 536, 508, 474, 485, 508, 506, 555, 500, 515, 531, 540, 514, 523, 532, 526, 529, 516, 503, 529, 505, 508, 491, 507, 501, 516, 509, 517, 501, 556, 518, 510, 502, 522, 446, 523, 476, 525, 555, 534, 519, 494, 511, 498, 522, 518, 525, 508, 496, 495, 515, 512, 522, 504, 507, 478, 511, 504] payload_rows_per_dst=[5616, 5522, 5628, 5514, 5596, 5472, 5478, 5508] dup_rows_per_dst=[2610, 2580, 2794, 2606, 2674, 2672, 2668, 2598] combine_rows_per_src=[5616, 5522, 5628, 5514, 5596, 5472, 5478, 5508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158842880 dispatch_meta_bytes=229320 combine_payload_bytes=158842880 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159072200 ingress_bytes=209174528 link_bytes=209174528 token_bytes=317915080 weight_movement_bytes=50331648 total_nvlink_bytes=368246728 weight_to_token_ratio=0.158318 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591576470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158392320 dispatch_meta_bytes=227136 combine_payload_bytes=158392320 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158619456 ingress_bytes=183558144 link_bytes=183558144 token_bytes=317011776 weight_movement_bytes=25165824 total_nvlink_bytes=342177600 weight_to_token_ratio=0.079385 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396665386e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160227328 dispatch_meta_bytes=229600 combine_payload_bytes=160227328 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160456928 ingress_bytes=210558976 link_bytes=210558976 token_bytes=320684256 weight_movement_bytes=50331648 total_nvlink_bytes=371015904 weight_to_token_ratio=0.156951 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602110520e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159621120 dispatch_meta_bytes=229392 combine_payload_bytes=159621120 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159850512 ingress_bytes=184786944 link_bytes=184786944 token_bytes=319471632 weight_movement_bytes=25165824 total_nvlink_bytes=344637456 weight_to_token_ratio=0.078773 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406015134e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160120832 dispatch_meta_bytes=229448 combine_payload_bytes=160120832 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160350280 ingress_bytes=185286656 link_bytes=185286656 token_bytes=320471112 weight_movement_bytes=25165824 total_nvlink_bytes=345636936 weight_to_token_ratio=0.078528 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409817365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=160071680 dispatch_meta_bytes=228800 combine_payload_bytes=160071680 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160300480 ingress_bytes=210403328 link_bytes=210403328 token_bytes=320372160 weight_movement_bytes=50331648 total_nvlink_bytes=370703808 weight_to_token_ratio=0.157104 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600926218e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158539776 dispatch_meta_bytes=228456 combine_payload_bytes=158539776 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158768232 ingress_bytes=234037248 link_bytes=234037248 token_bytes=317308008 weight_movement_bytes=75497472 total_nvlink_bytes=392805480 weight_to_token_ratio=0.237931 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.780753042e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44446 payload_rows_remote=38904 dup_rows=21090 dup_rows_remote=18484 dedup_ratio=0.321808 router_tokens_per_expert=[520, 523, 528, 465, 520, 516, 523, 496, 537, 520, 515, 491, 495, 516, 530, 539, 514, 524, 541, 509, 510, 518, 500, 496, 536, 500, 518, 522, 509, 479, 509, 501, 514, 481, 501, 523, 510, 504, 511, 522, 493, 563, 507, 517, 508, 505, 485, 528, 509, 514, 516, 520, 526, 464, 546, 532, 514, 501, 531, 497, 454, 514, 538, 500] payload_rows_per_dst=[5550, 5602, 5564, 5542, 5508, 5526, 5580, 5574] dup_rows_per_dst=[2632, 2684, 2660, 2606, 2624, 2686, 2674, 2524] combine_rows_per_src=[5550, 5602, 5564, 5542, 5508, 5526, 5580, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5105434552 prefetch_bytes=377487360 reduce_bytes=327155712 weight_movement_bytes=704643072 egress_bytes=2554552248 ingress_bytes=3255525376 link_bytes=3255525376 total_nvlink_bytes=5810077624 weight_to_token_ratio=0.138018 prefetch_share_of_ingress=0.115953 dedup_ratio=0.321182 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=13 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.548174160e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229552 combine_payload_bytes=159350784 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159580336 ingress_bytes=184516608 link_bytes=184516608 token_bytes=318931120 weight_movement_bytes=25165824 total_nvlink_bytes=344096944 weight_to_token_ratio=0.078907 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403958190e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38914 dup_rows=21050 dup_rows_remote=18364 dedup_ratio=0.321198 router_tokens_per_expert=[474, 512, 510, 553, 493, 487, 508, 524, 526, 481, 526, 489, 520, 501, 531, 537, 494, 552, 504, 493, 515, 489, 511, 516, 534, 536, 513, 529, 538, 520, 478, 496, 550, 507, 542, 521, 494, 491, 492, 494, 521, 502, 495, 501, 513, 499, 521, 489, 531, 495, 524, 494, 547, 510, 525, 503, 486, 505, 497, 516, 506, 563, 530, 514] payload_rows_per_dst=[5524, 5518, 5634, 5642, 5506, 5512, 5572, 5578] dup_rows_per_dst=[2598, 2704, 2514, 2646, 2676, 2570, 2686, 2656] combine_rows_per_src=[5524, 5518, 5634, 5642, 5506, 5512, 5572, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44462 payload_rows_remote=38752 dup_rows=21074 dup_rows_remote=18192 dedup_ratio=0.321564 router_tokens_per_expert=[490, 516, 495, 529, 505, 549, 491, 467, 506, 486, 477, 517, 533, 562, 526, 554, 536, 529, 539, 508, 501, 505, 476, 515, 521, 497, 558, 528, 536, 519, 469, 551, 492, 508, 479, 552, 502, 510, 511, 479, 495, 535, 526, 521, 534, 530, 495, 499, 538, 519, 497, 523, 490, 468, 471, 521, 520, 489, 473, 499, 562, 521, 498, 520] payload_rows_per_dst=[5494, 5628, 5628, 5520, 5502, 5618, 5362, 5710] dup_rows_per_dst=[2590, 2678, 2590, 2716, 2564, 2668, 2386, 2882] combine_rows_per_src=[5494, 5628, 5628, 5520, 5502, 5618, 5362, 5710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44236 payload_rows_remote=38678 dup_rows=21300 dup_rows_remote=18626 dedup_ratio=0.325012 router_tokens_per_expert=[477, 512, 526, 542, 517, 533, 488, 487, 485, 546, 499, 512, 503, 500, 500, 513, 494, 527, 501, 569, 519, 476, 483, 547, 464, 514, 519, 484, 493, 540, 511, 519, 541, 524, 526, 522, 509, 506, 521, 533, 551, 506, 512, 510, 490, 510, 532, 505, 495, 529, 532, 533, 518, 515, 505, 504, 515, 490, 513, 490, 531, 476, 498, 526] payload_rows_per_dst=[5500, 5536, 5566, 5452, 5562, 5558, 5584, 5478] dup_rows_per_dst=[2664, 2580, 2666, 2636, 2802, 2674, 2678, 2600] combine_rows_per_src=[5500, 5536, 5566, 5452, 5562, 5558, 5584, 5478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44518 payload_rows_remote=38786 dup_rows=21018 dup_rows_remote=18148 dedup_ratio=0.320709 router_tokens_per_expert=[531, 528, 494, 527, 543, 509, 530, 519, 481, 501, 535, 545, 465, 482, 533, 518, 502, 518, 567, 473, 489, 526, 511, 534, 543, 479, 505, 522, 510, 496, 494, 474, 466, 502, 469, 545, 515, 512, 505, 514, 517, 509, 533, 515, 524, 494, 509, 496, 522, 550, 532, 518, 511, 489, 560, 522, 560, 491, 499, 491, 503, 495, 500, 516] payload_rows_per_dst=[5732, 5546, 5526, 5460, 5596, 5562, 5572, 5524] dup_rows_per_dst=[2870, 2574, 2548, 2586, 2790, 2558, 2506, 2586] combine_rows_per_src=[5732, 5546, 5526, 5460, 5596, 5562, 5572, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44382 payload_rows_remote=38948 dup_rows=21154 dup_rows_remote=18624 dedup_ratio=0.322784 router_tokens_per_expert=[508, 493, 497, 537, 509, 524, 496, 481, 537, 521, 507, 490, 478, 480, 478, 491, 495, 534, 505, 502, 546, 561, 521, 547, 539, 465, 501, 532, 536, 506, 533, 528, 501, 501, 495, 515, 481, 562, 490, 493, 490, 505, 517, 502, 496, 522, 523, 499, 483, 538, 536, 523, 513, 532, 498, 526, 539, 523, 507, 521, 517, 518, 524, 500] payload_rows_per_dst=[5494, 5434, 5680, 5514, 5510, 5552, 5592, 5606] dup_rows_per_dst=[2596, 2530, 2742, 2766, 2566, 2556, 2706, 2692] combine_rows_per_src=[5494, 5434, 5680, 5514, 5510, 5552, 5592, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38930 dup_rows=21064 dup_rows_remote=18492 dedup_ratio=0.321411 router_tokens_per_expert=[514, 533, 516, 472, 538, 489, 492, 520, 535, 523, 517, 510, 498, 490, 494, 527, 506, 491, 500, 514, 503, 499, 526, 518, 499, 546, 497, 523, 488, 494, 528, 512, 540, 490, 521, 546, 526, 493, 482, 480, 476, 501, 534, 547, 516, 536, 493, 516, 498, 536, 535, 509, 531, 530, 506, 563, 488, 486, 510, 499, 515, 515, 491, 547] payload_rows_per_dst=[5584, 5570, 5542, 5508, 5514, 5582, 5652, 5520] dup_rows_per_dst=[2564, 2618, 2572, 2666, 2642, 2656, 2764, 2582] combine_rows_per_src=[5584, 5570, 5542, 5508, 5514, 5582, 5652, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5104501952 prefetch_bytes=301989888 reduce_bytes=201326592 weight_movement_bytes=503316480 egress_bytes=2554086592 ingress_bytes=3053731840 link_bytes=3053731840 total_nvlink_bytes=5607818432 weight_to_token_ratio=0.098602 prefetch_share_of_ingress=0.098892 dedup_ratio=0.321472 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=8 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042969 gemm_inner_pad_upper_bound=19812 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.452210682e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159391744 dispatch_meta_bytes=229112 combine_payload_bytes=159391744 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159620856 ingress_bytes=234889216 link_bytes=234889216 token_bytes=319012600 weight_movement_bytes=75497472 total_nvlink_bytes=394510072 weight_to_token_ratio=0.236660 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787235534e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158728192 dispatch_meta_bytes=227776 combine_payload_bytes=158728192 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158955968 ingress_bytes=183894016 link_bytes=183894016 token_bytes=317684160 weight_movement_bytes=25165824 total_nvlink_bytes=342849984 weight_to_token_ratio=0.079216 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399220984e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158425088 dispatch_meta_bytes=229216 combine_payload_bytes=158425088 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158654304 ingress_bytes=208756736 link_bytes=208756736 token_bytes=317079392 weight_movement_bytes=50331648 total_nvlink_bytes=367411040 weight_to_token_ratio=0.158735 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.588397556e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158867456 dispatch_meta_bytes=227736 combine_payload_bytes=158867456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159095192 ingress_bytes=184033280 link_bytes=184033280 token_bytes=317962648 weight_movement_bytes=25165824 total_nvlink_bytes=343128472 weight_to_token_ratio=0.079147 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400280622e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159531008 dispatch_meta_bytes=230288 combine_payload_bytes=159531008 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159761296 ingress_bytes=184696832 link_bytes=184696832 token_bytes=319292304 weight_movement_bytes=25165824 total_nvlink_bytes=344458128 weight_to_token_ratio=0.078818 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405329486e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5109268416 prefetch_bytes=352321536 reduce_bytes=478150656 weight_movement_bytes=830472192 egress_bytes=2556469184 ingress_bytes=3383271424 link_bytes=3383271424 total_nvlink_bytes=5939740608 weight_to_token_ratio=0.162542 prefetch_share_of_ingress=0.104136 dedup_ratio=0.320971 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=19 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.608924149e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159457280 dispatch_meta_bytes=229688 combine_payload_bytes=159457280 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159686968 ingress_bytes=209788928 link_bytes=209788928 token_bytes=319144248 weight_movement_bytes=50331648 total_nvlink_bytes=369475896 weight_to_token_ratio=0.157708 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596251344e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5100707176 prefetch_bytes=377487360 reduce_bytes=301989888 weight_movement_bytes=679477248 egress_bytes=2552184168 ingress_bytes=3228000256 link_bytes=3228000256 total_nvlink_bytes=5780184424 weight_to_token_ratio=0.133212 prefetch_share_of_ingress=0.116942 dedup_ratio=0.320978 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=12 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.535084512e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5114593232 prefetch_bytes=301989888 reduce_bytes=478150656 weight_movement_bytes=780140544 egress_bytes=2559131600 ingress_bytes=3335602176 link_bytes=3335602176 total_nvlink_bytes=5894733776 weight_to_token_ratio=0.152532 prefetch_share_of_ingress=0.090535 dedup_ratio=0.320244 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=19 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=19812 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.586254905e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5104771744 prefetch_bytes=301989888 reduce_bytes=352321536 weight_movement_bytes=654311424 egress_bytes=2554217120 ingress_bytes=3204866048 link_bytes=3204866048 total_nvlink_bytes=5759083168 weight_to_token_ratio=0.128176 prefetch_share_of_ingress=0.094229 dedup_ratio=0.320644 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=14 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.043658 gemm_inner_pad_upper_bound=19812 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.524082975e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5105026552 prefetch_bytes=377487360 reduce_bytes=150994944 weight_movement_bytes=528482304 egress_bytes=2554349048 ingress_bytes=3079159808 link_bytes=3079159808 total_nvlink_bytes=5633508856 weight_to_token_ratio=0.103522 prefetch_share_of_ingress=0.122594 dedup_ratio=0.321543 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=6 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042509 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.464303023e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5108369848 prefetch_bytes=352321536 reduce_bytes=452984832 weight_movement_bytes=805306368 egress_bytes=2556021176 ingress_bytes=3357655040 link_bytes=3357655040 total_nvlink_bytes=5913676216 weight_to_token_ratio=0.157644 prefetch_share_of_ingress=0.104931 dedup_ratio=0.321228 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=18 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.596742206e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none + [2026-09-21 08:32:50] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 211.8 | throughput per GPU (TFLOP/s/GPU): 152.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.711792E+00 | loss scale: 1.0 | grad norm: 3.231 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44508 payload_rows_remote=38798 dup_rows=21028 dup_rows_remote=18302 dedup_ratio=0.320862 router_tokens_per_expert=[508, 521, 539, 495, 516, 502, 477, 502, 479, 554, 519, 509, 530, 575, 479, 549, 529, 479, 519, 588, 506, 512, 546, 539, 562, 510, 503, 516, 499, 531, 512, 544, 542, 522, 503, 505, 498, 506, 505, 518, 502, 501, 525, 488, 500, 496, 510, 478, 478, 530, 497, 492, 510, 510, 524, 506, 494, 505, 498, 490, 508, 489, 498, 491] payload_rows_per_dst=[5568, 5608, 5710, 5614, 5568, 5488, 5476, 5476] dup_rows_per_dst=[2552, 2780, 2726, 2740, 2630, 2512, 2618, 2470] combine_rows_per_src=[5568, 5608, 5710, 5614, 5568, 5488, 5476, 5476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44434 payload_rows_remote=38846 dup_rows=21102 dup_rows_remote=18458 dedup_ratio=0.321991 router_tokens_per_expert=[525, 509, 526, 502, 522, 522, 501, 501, 522, 526, 514, 487, 522, 510, 523, 537, 485, 508, 465, 494, 535, 538, 477, 518, 529, 518, 502, 517, 502, 516, 522, 530, 570, 477, 534, 486, 498, 486, 504, 467, 562, 522, 514, 519, 527, 481, 521, 512, 501, 535, 505, 529, 483, 512, 555, 496, 478, 487, 506, 545, 509, 492, 535, 515] payload_rows_per_dst=[5528, 5588, 5522, 5536, 5568, 5582, 5588, 5522] dup_rows_per_dst=[2688, 2694, 2518, 2736, 2476, 2734, 2644, 2612] combine_rows_per_src=[5528, 5588, 5522, 5536, 5568, 5582, 5588, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44752 payload_rows_remote=39154 dup_rows=20784 dup_rows_remote=18132 dedup_ratio=0.317139 router_tokens_per_expert=[501, 555, 475, 496, 516, 506, 541, 533, 497, 517, 522, 523, 517, 518, 553, 533, 513, 471, 499, 523, 497, 527, 506, 502, 483, 506, 498, 521, 492, 479, 521, 544, 513, 496, 513, 530, 506, 553, 498, 516, 532, 525, 518, 488, 543, 486, 473, 513, 478, 512, 498, 546, 514, 473, 481, 541, 520, 531, 517, 527, 559, 501, 507, 475] payload_rows_per_dst=[5564, 5682, 5634, 5586, 5598, 5550, 5518, 5620] dup_rows_per_dst=[2682, 2678, 2442, 2502, 2652, 2606, 2568, 2654] combine_rows_per_src=[5564, 5682, 5634, 5586, 5598, 5550, 5518, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44654 payload_rows_remote=39034 dup_rows=20882 dup_rows_remote=18288 dedup_ratio=0.318634 router_tokens_per_expert=[507, 510, 530, 556, 523, 503, 513, 491, 510, 525, 508, 494, 545, 508, 465, 572, 510, 464, 478, 477, 508, 522, 540, 523, 527, 508, 487, 522, 538, 498, 515, 512, 510, 521, 494, 504, 489, 515, 544, 502, 509, 526, 493, 517, 520, 494, 523, 552, 497, 488, 490, 521, 541, 515, 521, 516, 490, 517, 499, 499, 506, 544, 481, 541] payload_rows_per_dst=[5564, 5598, 5438, 5620, 5510, 5636, 5684, 5604] dup_rows_per_dst=[2702, 2656, 2606, 2594, 2648, 2632, 2494, 2550] combine_rows_per_src=[5564, 5598, 5438, 5620, 5510, 5636, 5684, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44684 payload_rows_remote=39046 dup_rows=20852 dup_rows_remote=18312 dedup_ratio=0.318176 router_tokens_per_expert=[510, 535, 500, 495, 481, 526, 529, 540, 525, 500, 510, 519, 468, 535, 502, 530, 487, 519, 548, 528, 502, 488, 501, 510, 514, 498, 532, 501, 529, 497, 514, 529, 478, 502, 487, 544, 508, 491, 538, 522, 519, 532, 550, 540, 510, 475, 547, 522, 490, 516, 501, 528, 513, 524, 482, 505, 522, 502, 500, 489, 501, 544, 473, 511] payload_rows_per_dst=[5564, 5638, 5596, 5588, 5588, 5600, 5582, 5528] dup_rows_per_dst=[2668, 2540, 2570, 2640, 2552, 2790, 2536, 2556] combine_rows_per_src=[5564, 5638, 5596, 5588, 5588, 5600, 5582, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38656 dup_rows=21166 dup_rows_remote=18294 dedup_ratio=0.322968 router_tokens_per_expert=[501, 499, 505, 517, 504, 527, 509, 494, 433, 514, 544, 492, 521, 455, 514, 552, 501, 546, 524, 562, 526, 527, 491, 491, 491, 499, 495, 491, 525, 522, 524, 494, 472, 535, 516, 490, 490, 514, 536, 514, 545, 567, 530, 518, 532, 494, 555, 527, 537, 490, 500, 473, 522, 470, 538, 465, 469, 528, 529, 530, 478, 532, 553, 529] payload_rows_per_dst=[5558, 5410, 5488, 5422, 5618, 5562, 5598, 5714] dup_rows_per_dst=[2544, 2554, 2578, 2482, 2730, 2718, 2688, 2872] combine_rows_per_src=[5558, 5410, 5488, 5422, 5618, 5562, 5598, 5714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38988 dup_rows=21108 dup_rows_remote=18584 dedup_ratio=0.322083 router_tokens_per_expert=[531, 480, 567, 523, 493, 505, 509, 545, 493, 521, 486, 515, 486, 494, 503, 505, 512, 555, 505, 487, 515, 518, 525, 515, 504, 481, 509, 565, 544, 499, 497, 512, 486, 549, 494, 524, 517, 490, 505, 558, 505, 487, 480, 492, 540, 490, 485, 503, 518, 513, 506, 574, 497, 547, 529, 523, 488, 496, 555, 490, 524, 494, 509, 501] payload_rows_per_dst=[5618, 5410, 5556, 5640, 5570, 5440, 5694, 5500] dup_rows_per_dst=[2688, 2596, 2708, 2582, 2676, 2524, 2720, 2614] combine_rows_per_src=[5618, 5410, 5556, 5640, 5570, 5440, 5694, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158916608 dispatch_meta_bytes=228400 combine_payload_bytes=158916608 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159145008 ingress_bytes=209248256 link_bytes=209248256 token_bytes=318061616 weight_movement_bytes=50331648 total_nvlink_bytes=368393264 weight_to_token_ratio=0.158245 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592137455e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=229216 combine_payload_bytes=159113216 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159342432 ingress_bytes=184279040 link_bytes=184279040 token_bytes=318455648 weight_movement_bytes=25165824 total_nvlink_bytes=343621472 weight_to_token_ratio=0.079025 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402150572e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160374784 dispatch_meta_bytes=229144 combine_payload_bytes=160374784 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160603928 ingress_bytes=185540608 link_bytes=185540608 token_bytes=320978712 weight_movement_bytes=25165824 total_nvlink_bytes=346144536 weight_to_token_ratio=0.078403 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411749647e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=229288 combine_payload_bytes=159883264 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160112552 ingress_bytes=210214912 link_bytes=210214912 token_bytes=319995816 weight_movement_bytes=50331648 total_nvlink_bytes=370327464 weight_to_token_ratio=0.157288 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599492590e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44686 payload_rows_remote=39146 dup_rows=20850 dup_rows_remote=18342 dedup_ratio=0.318146 router_tokens_per_expert=[487, 530, 487, 455, 529, 497, 509, 530, 480, 478, 500, 536, 544, 528, 492, 494, 508, 516, 528, 485, 556, 541, 551, 537, 546, 513, 502, 497, 555, 501, 528, 485, 526, 494, 477, 522, 493, 488, 527, 549, 492, 539, 530, 526, 479, 477, 512, 526, 550, 521, 515, 489, 513, 501, 487, 488, 506, 509, 551, 508, 502, 545, 488, 513] payload_rows_per_dst=[5540, 5550, 5624, 5624, 5568, 5602, 5542, 5636] dup_rows_per_dst=[2508, 2554, 2820, 2630, 2584, 2560, 2586, 2608] combine_rows_per_src=[5540, 5550, 5624, 5624, 5568, 5602, 5542, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159932416 dispatch_meta_bytes=229432 combine_payload_bytes=159932416 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=160161848 ingress_bytes=260595712 link_bytes=260595712 token_bytes=320094264 weight_movement_bytes=100663296 total_nvlink_bytes=420757560 weight_to_token_ratio=0.314480 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.982832267e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158334976 dispatch_meta_bytes=227800 combine_payload_bytes=158334976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158562776 ingress_bytes=183500800 link_bytes=183500800 token_bytes=316897752 weight_movement_bytes=25165824 total_nvlink_bytes=342063576 weight_to_token_ratio=0.079413 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.396229065e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=230288 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159925136 ingress_bytes=210026496 link_bytes=210026496 token_bytes=319619984 weight_movement_bytes=50331648 total_nvlink_bytes=369951632 weight_to_token_ratio=0.157473 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598058962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160342016 dispatch_meta_bytes=229952 combine_payload_bytes=160342016 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=160571968 ingress_bytes=185507840 link_bytes=185507840 token_bytes=320913984 weight_movement_bytes=25165824 total_nvlink_bytes=346079808 weight_to_token_ratio=0.078419 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411500320e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44452 payload_rows_remote=38970 dup_rows=21084 dup_rows_remote=18462 dedup_ratio=0.321716 router_tokens_per_expert=[446, 520, 536, 498, 527, 536, 526, 507, 536, 523, 517, 514, 509, 513, 506, 539, 524, 507, 480, 526, 488, 514, 512, 501, 553, 500, 510, 505, 490, 485, 511, 505, 508, 461, 488, 521, 502, 509, 506, 512, 547, 534, 530, 515, 503, 468, 533, 515, 519, 525, 510, 478, 533, 530, 524, 519, 509, 553, 503, 507, 504, 499, 524, 515] payload_rows_per_dst=[5530, 5604, 5482, 5498, 5488, 5620, 5628, 5602] dup_rows_per_dst=[2662, 2710, 2622, 2620, 2526, 2670, 2648, 2626] combine_rows_per_src=[5530, 5604, 5482, 5498, 5488, 5620, 5628, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44776 payload_rows_remote=39132 dup_rows=20760 dup_rows_remote=18182 dedup_ratio=0.316772 router_tokens_per_expert=[513, 529, 523, 540, 520, 478, 510, 514, 522, 548, 533, 504, 519, 487, 524, 496, 486, 540, 530, 527, 526, 516, 566, 489, 526, 503, 494, 513, 513, 532, 517, 527, 512, 532, 550, 506, 474, 507, 530, 500, 479, 488, 480, 517, 519, 505, 502, 478, 506, 502, 495, 480, 537, 528, 506, 514, 524, 511, 509, 510, 488, 511, 523, 480] payload_rows_per_dst=[5646, 5642, 5664, 5638, 5644, 5478, 5526, 5538] dup_rows_per_dst=[2608, 2624, 2696, 2612, 2578, 2458, 2610, 2574] combine_rows_per_src=[5646, 5642, 5664, 5638, 5644, 5478, 5526, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44532 payload_rows_remote=38964 dup_rows=21004 dup_rows_remote=18432 dedup_ratio=0.320496 router_tokens_per_expert=[537, 494, 501, 539, 545, 535, 495, 497, 506, 503, 499, 532, 520, 490, 474, 531, 505, 523, 474, 502, 489, 513, 536, 495, 529, 497, 538, 520, 492, 479, 519, 528, 508, 496, 507, 491, 512, 505, 527, 517, 505, 547, 545, 530, 488, 506, 512, 526, 471, 500, 484, 550, 529, 497, 504, 535, 540, 550, 494, 529, 490, 533, 489, 514] payload_rows_per_dst=[5598, 5518, 5454, 5574, 5556, 5636, 5568, 5628] dup_rows_per_dst=[2688, 2592, 2620, 2630, 2570, 2682, 2572, 2650] combine_rows_per_src=[5598, 5518, 5454, 5574, 5556, 5636, 5568, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44462 payload_rows_remote=38848 dup_rows=21074 dup_rows_remote=18486 dedup_ratio=0.321564 router_tokens_per_expert=[520, 478, 483, 479, 549, 539, 506, 515, 512, 507, 473, 538, 484, 514, 467, 544, 520, 528, 517, 505, 506, 562, 510, 490, 525, 515, 506, 513, 522, 499, 537, 484, 478, 515, 521, 541, 518, 506, 490, 529, 489, 521, 515, 513, 505, 509, 508, 508, 510, 522, 540, 521, 503, 538, 514, 527, 497, 517, 525, 505, 509, 531, 504, 492] payload_rows_per_dst=[5538, 5440, 5592, 5614, 5564, 5548, 5672, 5494] dup_rows_per_dst=[2600, 2638, 2684, 2588, 2632, 2588, 2678, 2666] combine_rows_per_src=[5538, 5440, 5592, 5614, 5564, 5548, 5672, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=38996 dup_rows=21012 dup_rows_remote=18352 dedup_ratio=0.320618 router_tokens_per_expert=[534, 513, 525, 499, 494, 500, 516, 506, 458, 524, 507, 556, 513, 516, 575, 503, 497, 498, 513, 495, 522, 484, 503, 525, 519, 498, 502, 520, 516, 512, 535, 515, 515, 501, 488, 553, 501, 505, 487, 520, 489, 527, 501, 534, 506, 523, 514, 494, 538, 539, 487, 494, 516, 528, 510, 519, 554, 469, 519, 529, 512, 519, 509, 475] payload_rows_per_dst=[5480, 5618, 5502, 5498, 5662, 5662, 5574, 5528] dup_rows_per_dst=[2368, 2666, 2576, 2522, 2804, 2728, 2688, 2660] combine_rows_per_src=[5480, 5618, 5502, 5498, 5662, 5662, 5574, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44320 payload_rows_remote=38758 dup_rows=21216 dup_rows_remote=18532 dedup_ratio=0.323730 router_tokens_per_expert=[532, 452, 509, 526, 528, 529, 493, 522, 493, 554, 497, 517, 531, 488, 518, 525, 494, 518, 508, 523, 533, 516, 511, 533, 519, 502, 473, 567, 490, 536, 531, 477, 508, 506, 515, 565, 543, 505, 488, 499, 516, 523, 492, 482, 476, 517, 494, 532, 476, 544, 515, 514, 485, 508, 495, 537, 531, 489, 506, 512, 537, 496, 506, 511] payload_rows_per_dst=[5558, 5562, 5552, 5520, 5580, 5486, 5496, 5566] dup_rows_per_dst=[2624, 2684, 2720, 2670, 2678, 2578, 2652, 2610] combine_rows_per_src=[5558, 5562, 5552, 5520, 5580, 5486, 5496, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44504 payload_rows_remote=38918 dup_rows=21032 dup_rows_remote=18392 dedup_ratio=0.320923 router_tokens_per_expert=[518, 535, 516, 533, 515, 544, 513, 516, 506, 525, 505, 466, 501, 516, 536, 493, 516, 506, 500, 515, 517, 496, 538, 514, 498, 542, 511, 492, 507, 471, 495, 546, 537, 446, 525, 532, 520, 467, 478, 516, 524, 537, 476, 529, 487, 519, 532, 509, 525, 491, 503, 501, 523, 507, 502, 561, 517, 523, 517, 504, 510, 512, 510, 526] payload_rows_per_dst=[5652, 5464, 5592, 5554, 5504, 5586, 5568, 5584] dup_rows_per_dst=[2728, 2632, 2612, 2570, 2538, 2640, 2658, 2654] combine_rows_per_src=[5652, 5464, 5592, 5554, 5504, 5586, 5568, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159621120 dispatch_meta_bytes=229728 combine_payload_bytes=159621120 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159850848 ingress_bytes=209952768 link_bytes=209952768 token_bytes=319471968 weight_movement_bytes=50331648 total_nvlink_bytes=369803616 weight_to_token_ratio=0.157546 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597497977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=160284672 dispatch_meta_bytes=229256 combine_payload_bytes=160284672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160513928 ingress_bytes=185450496 link_bytes=185450496 token_bytes=320798600 weight_movement_bytes=25165824 total_nvlink_bytes=345964424 weight_to_token_ratio=0.078447 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411063998e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229584 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159826128 ingress_bytes=209928192 link_bytes=209928192 token_bytes=319422672 weight_movement_bytes=50331648 total_nvlink_bytes=369754320 weight_to_token_ratio=0.157571 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597310982e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159121408 dispatch_meta_bytes=229336 combine_payload_bytes=159121408 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159350744 ingress_bytes=209453056 link_bytes=209453056 token_bytes=318472152 weight_movement_bytes=50331648 total_nvlink_bytes=368803800 weight_to_token_ratio=0.158041 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593695747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159727616 dispatch_meta_bytes=229392 combine_payload_bytes=159727616 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159957008 ingress_bytes=210059264 link_bytes=210059264 token_bytes=319684624 weight_movement_bytes=50331648 total_nvlink_bytes=370016272 weight_to_token_ratio=0.157442 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598308289e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158752768 dispatch_meta_bytes=229160 combine_payload_bytes=158752768 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158981928 ingress_bytes=209084416 link_bytes=209084416 token_bytes=317734696 weight_movement_bytes=50331648 total_nvlink_bytes=368066344 weight_to_token_ratio=0.158408 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590890822e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44398 payload_rows_remote=38800 dup_rows=21138 dup_rows_remote=18480 dedup_ratio=0.322540 router_tokens_per_expert=[515, 537, 525, 479, 541, 502, 509, 486, 482, 533, 491, 523, 471, 515, 518, 538, 541, 496, 501, 530, 480, 542, 519, 509, 547, 539, 503, 499, 466, 536, 553, 510, 506, 512, 520, 546, 484, 521, 502, 515, 518, 524, 494, 480, 517, 480, 535, 540, 484, 524, 514, 490, 508, 526, 530, 465, 498, 480, 502, 565, 533, 502, 504, 513] payload_rows_per_dst=[5598, 5546, 5496, 5668, 5588, 5448, 5470, 5584] dup_rows_per_dst=[2658, 2596, 2672, 2760, 2624, 2728, 2528, 2572] combine_rows_per_src=[5598, 5546, 5496, 5668, 5588, 5448, 5470, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=229240 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159637368 ingress_bytes=184573952 link_bytes=184573952 token_bytes=319045496 weight_movement_bytes=25165824 total_nvlink_bytes=344211320 weight_to_token_ratio=0.078878 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404394511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158924800 dispatch_meta_bytes=229120 combine_payload_bytes=158924800 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159153920 ingress_bytes=209256448 link_bytes=209256448 token_bytes=318078720 weight_movement_bytes=50331648 total_nvlink_bytes=368410368 weight_to_token_ratio=0.158236 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592199787e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44600 payload_rows_remote=39098 dup_rows=20936 dup_rows_remote=18292 dedup_ratio=0.319458 router_tokens_per_expert=[511, 515, 502, 522, 538, 509, 538, 506, 500, 487, 507, 482, 528, 508, 535, 542, 509, 492, 501, 523, 503, 518, 520, 507, 506, 527, 527, 472, 515, 537, 525, 495, 531, 514, 522, 475, 527, 553, 545, 515, 474, 508, 510, 522, 496, 501, 504, 497, 480, 520, 520, 489, 483, 550, 472, 529, 460, 510, 514, 516, 542, 533, 522, 527] payload_rows_per_dst=[5586, 5634, 5502, 5566, 5640, 5500, 5538, 5634] dup_rows_per_dst=[2696, 2544, 2644, 2642, 2724, 2524, 2548, 2614] combine_rows_per_src=[5586, 5634, 5502, 5566, 5640, 5500, 5538, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44736 payload_rows_remote=39134 dup_rows=20800 dup_rows_remote=18166 dedup_ratio=0.317383 router_tokens_per_expert=[529, 522, 471, 489, 505, 488, 544, 504, 510, 530, 525, 528, 500, 515, 508, 515, 492, 497, 521, 517, 485, 500, 532, 495, 534, 513, 538, 491, 493, 488, 524, 480, 524, 537, 515, 495, 507, 488, 527, 525, 560, 527, 537, 538, 542, 517, 480, 495, 548, 493, 507, 498, 534, 497, 493, 513, 565, 534, 488, 516, 485, 492, 503, 505] payload_rows_per_dst=[5508, 5616, 5594, 5606, 5602, 5648, 5568, 5594] dup_rows_per_dst=[2596, 2646, 2484, 2516, 2634, 2744, 2598, 2582] combine_rows_per_src=[5508, 5616, 5594, 5606, 5602, 5648, 5568, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44332 payload_rows_remote=38782 dup_rows=21204 dup_rows_remote=18570 dedup_ratio=0.323547 router_tokens_per_expert=[529, 514, 499, 496, 473, 500, 527, 541, 519, 483, 521, 503, 492, 516, 519, 491, 567, 510, 530, 513, 493, 521, 491, 495, 574, 519, 493, 501, 547, 543, 516, 543, 482, 515, 531, 492, 513, 510, 502, 492, 508, 489, 511, 524, 503, 477, 517, 540, 525, 493, 525, 520, 512, 486, 503, 528, 511, 552, 471, 480, 526, 489, 531, 531] payload_rows_per_dst=[5578, 5458, 5510, 5644, 5526, 5552, 5550, 5514] dup_rows_per_dst=[2580, 2630, 2730, 2828, 2548, 2586, 2634, 2668] combine_rows_per_src=[5578, 5458, 5510, 5644, 5526, 5552, 5550, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44750 payload_rows_remote=39088 dup_rows=20786 dup_rows_remote=18086 dedup_ratio=0.317169 router_tokens_per_expert=[503, 516, 489, 537, 519, 496, 502, 501, 518, 534, 493, 494, 503, 484, 550, 563, 535, 489, 515, 549, 515, 497, 528, 567, 557, 508, 525, 538, 483, 523, 501, 546, 506, 515, 496, 513, 494, 493, 502, 542, 492, 484, 520, 469, 516, 549, 525, 499, 513, 520, 484, 490, 526, 539, 482, 496, 561, 465, 524, 492, 492, 509, 473, 509] payload_rows_per_dst=[5546, 5646, 5624, 5662, 5564, 5560, 5592, 5556] dup_rows_per_dst=[2580, 2632, 2766, 2700, 2558, 2548, 2508, 2494] combine_rows_per_src=[5546, 5646, 5624, 5662, 5564, 5560, 5592, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44520 payload_rows_remote=38962 dup_rows=21016 dup_rows_remote=18410 dedup_ratio=0.320679 router_tokens_per_expert=[524, 552, 530, 521, 520, 522, 514, 483, 524, 527, 519, 514, 538, 517, 513, 500, 522, 538, 507, 522, 475, 509, 493, 541, 475, 544, 509, 522, 482, 509, 525, 516, 482, 475, 514, 496, 536, 496, 481, 519, 518, 480, 538, 540, 514, 478, 513, 516, 482, 543, 506, 542, 477, 519, 506, 515, 464, 500, 509, 526, 501, 514, 547, 514] payload_rows_per_dst=[5630, 5418, 5400, 5476, 5630, 5764, 5644, 5558] dup_rows_per_dst=[2666, 2400, 2554, 2414, 2720, 2916, 2740, 2606] combine_rows_per_src=[5630, 5418, 5400, 5476, 5630, 5764, 5644, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=38942 dup_rows=21096 dup_rows_remote=18406 dedup_ratio=0.321899 router_tokens_per_expert=[517, 553, 491, 488, 547, 507, 508, 484, 521, 480, 540, 489, 524, 493, 498, 549, 489, 538, 533, 530, 533, 502, 501, 531, 466, 505, 507, 513, 521, 505, 482, 524, 498, 513, 534, 507, 502, 502, 524, 521, 511, 504, 510, 490, 507, 476, 537, 513, 488, 539, 484, 541, 498, 504, 499, 526, 554, 499, 525, 501, 529, 502, 533, 528] payload_rows_per_dst=[5572, 5498, 5592, 5518, 5578, 5546, 5512, 5624] dup_rows_per_dst=[2618, 2690, 2722, 2528, 2624, 2550, 2646, 2718] combine_rows_per_src=[5572, 5498, 5592, 5518, 5578, 5546, 5512, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=160145408 dispatch_meta_bytes=229560 combine_payload_bytes=160145408 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160374968 ingress_bytes=235642880 link_bytes=235642880 token_bytes=320520376 weight_movement_bytes=75497472 total_nvlink_bytes=396017848 weight_to_token_ratio=0.235547 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792970046e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158851072 dispatch_meta_bytes=229408 combine_payload_bytes=158851072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159080480 ingress_bytes=184016896 link_bytes=184016896 token_bytes=317931552 weight_movement_bytes=25165824 total_nvlink_bytes=343097376 weight_to_token_ratio=0.079155 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400155959e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=160292864 dispatch_meta_bytes=229200 combine_payload_bytes=160292864 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160522064 ingress_bytes=185458688 link_bytes=185458688 token_bytes=320814928 weight_movement_bytes=25165824 total_nvlink_bytes=345980752 weight_to_token_ratio=0.078443 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411126330e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44590 payload_rows_remote=39062 dup_rows=20946 dup_rows_remote=18410 dedup_ratio=0.319611 router_tokens_per_expert=[538, 500, 489, 520, 533, 503, 496, 544, 502, 552, 496, 524, 532, 541, 523, 499, 526, 548, 452, 521, 485, 511, 510, 540, 481, 544, 538, 497, 498, 520, 523, 458, 493, 511, 515, 507, 496, 484, 516, 517, 516, 501, 494, 513, 487, 517, 484, 520, 502, 507, 540, 505, 462, 489, 570, 487, 540, 466, 545, 520, 540, 542, 527, 511] payload_rows_per_dst=[5570, 5678, 5560, 5570, 5572, 5528, 5456, 5656] dup_rows_per_dst=[2676, 2660, 2626, 2548, 2506, 2536, 2668, 2726] combine_rows_per_src=[5570, 5678, 5560, 5570, 5572, 5528, 5456, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=160104448 dispatch_meta_bytes=228696 combine_payload_bytes=160104448 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160333144 ingress_bytes=210436096 link_bytes=210436096 token_bytes=320437592 weight_movement_bytes=50331648 total_nvlink_bytes=370769240 weight_to_token_ratio=0.157072 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601175545e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159588352 dispatch_meta_bytes=229488 combine_payload_bytes=159588352 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159817840 ingress_bytes=209920000 link_bytes=209920000 token_bytes=319406192 weight_movement_bytes=50331648 total_nvlink_bytes=369737840 weight_to_token_ratio=0.157579 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597248651e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=229392 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159735824 ingress_bytes=209838080 link_bytes=209838080 token_bytes=319242256 weight_movement_bytes=50331648 total_nvlink_bytes=369573904 weight_to_token_ratio=0.157660 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596625334e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44284 payload_rows_remote=38730 dup_rows=21252 dup_rows_remote=18672 dedup_ratio=0.324280 router_tokens_per_expert=[513, 513, 535, 498, 492, 483, 498, 535, 518, 557, 515, 475, 509, 504, 545, 512, 524, 547, 502, 475, 502, 530, 511, 523, 499, 505, 540, 538, 551, 498, 498, 530, 510, 481, 475, 498, 530, 526, 502, 533, 478, 461, 506, 473, 529, 537, 524, 509, 559, 517, 486, 502, 552, 516, 490, 545, 540, 484, 527, 508, 517, 494, 492, 492] payload_rows_per_dst=[5554, 5594, 5536, 5566, 5486, 5488, 5618, 5442] dup_rows_per_dst=[2580, 2792, 2692, 2752, 2624, 2546, 2716, 2550] combine_rows_per_src=[5554, 5594, 5536, 5566, 5486, 5488, 5618, 5442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159997952 dispatch_meta_bytes=229888 combine_payload_bytes=159997952 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160227840 ingress_bytes=185163776 link_bytes=185163776 token_bytes=320225792 weight_movement_bytes=25165824 total_nvlink_bytes=345391616 weight_to_token_ratio=0.078588 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408882390e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38890 dup_rows=21050 dup_rows_remote=18380 dedup_ratio=0.321198 router_tokens_per_expert=[535, 481, 518, 496, 523, 505, 542, 531, 522, 514, 533, 494, 505, 513, 524, 510, 489, 508, 549, 493, 561, 528, 512, 493, 536, 467, 508, 497, 541, 545, 490, 547, 524, 521, 516, 503, 520, 503, 527, 484, 501, 515, 514, 492, 496, 503, 518, 496, 518, 535, 470, 516, 548, 501, 499, 532, 509, 494, 494, 511, 515, 499, 501, 483] payload_rows_per_dst=[5614, 5572, 5596, 5562, 5508, 5542, 5628, 5464] dup_rows_per_dst=[2648, 2658, 2670, 2700, 2688, 2528, 2610, 2548] combine_rows_per_src=[5614, 5572, 5596, 5562, 5508, 5542, 5628, 5464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=158638080 dispatch_meta_bytes=229608 combine_payload_bytes=158638080 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158867688 ingress_bytes=208969728 link_bytes=208969728 token_bytes=317505768 weight_movement_bytes=50331648 total_nvlink_bytes=367837416 weight_to_token_ratio=0.158522 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590018179e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44458 payload_rows_remote=38884 dup_rows=21078 dup_rows_remote=18430 dedup_ratio=0.321625 router_tokens_per_expert=[536, 471, 506, 503, 568, 504, 531, 450, 538, 520, 510, 484, 505, 488, 513, 542, 521, 506, 531, 492, 508, 516, 516, 535, 512, 521, 501, 505, 477, 512, 517, 514, 533, 503, 517, 538, 512, 548, 493, 456, 516, 516, 483, 530, 522, 502, 546, 508, 487, 524, 505, 514, 515, 518, 505, 543, 484, 479, 532, 535, 478, 525, 542, 506] payload_rows_per_dst=[5560, 5538, 5572, 5564, 5564, 5592, 5574, 5494] dup_rows_per_dst=[2578, 2662, 2678, 2554, 2636, 2654, 2648, 2668] combine_rows_per_src=[5560, 5538, 5572, 5564, 5564, 5592, 5574, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44424 payload_rows_remote=38914 dup_rows=21112 dup_rows_remote=18444 dedup_ratio=0.322144 router_tokens_per_expert=[513, 527, 506, 507, 534, 515, 559, 483, 506, 502, 544, 531, 506, 529, 500, 508, 511, 477, 489, 516, 544, 475, 502, 518, 501, 528, 488, 567, 521, 529, 549, 514, 466, 473, 559, 490, 549, 516, 484, 552, 504, 508, 483, 533, 551, 530, 527, 503, 549, 526, 504, 497, 543, 515, 523, 514, 471, 459, 484, 502, 479, 451, 493, 531] payload_rows_per_dst=[5562, 5610, 5494, 5680, 5510, 5592, 5686, 5290] dup_rows_per_dst=[2726, 2642, 2570, 2714, 2668, 2686, 2656, 2450] combine_rows_per_src=[5562, 5610, 5494, 5680, 5510, 5592, 5686, 5290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44598 payload_rows_remote=38998 dup_rows=20938 dup_rows_remote=18396 dedup_ratio=0.319489 router_tokens_per_expert=[539, 529, 489, 509, 551, 483, 486, 514, 504, 501, 537, 539, 500, 531, 526, 528, 511, 518, 514, 512, 479, 513, 503, 484, 544, 513, 518, 467, 543, 488, 512, 486, 542, 508, 497, 492, 522, 510, 486, 489, 505, 520, 509, 531, 525, 523, 477, 548, 478, 505, 557, 492, 506, 520, 470, 492, 559, 505, 477, 541, 537, 510, 515, 549] payload_rows_per_dst=[5574, 5638, 5526, 5600, 5482, 5600, 5520, 5658] dup_rows_per_dst=[2626, 2694, 2542, 2542, 2610, 2676, 2520, 2728] combine_rows_per_src=[5574, 5638, 5526, 5600, 5482, 5600, 5520, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44480 payload_rows_remote=38730 dup_rows=21056 dup_rows_remote=18104 dedup_ratio=0.321289 router_tokens_per_expert=[559, 518, 548, 476, 483, 502, 549, 487, 514, 509, 490, 510, 522, 520, 526, 487, 519, 549, 483, 487, 517, 526, 518, 509, 498, 522, 557, 501, 517, 476, 516, 526, 532, 489, 519, 487, 534, 489, 514, 484, 496, 506, 479, 537, 509, 531, 514, 476, 524, 525, 492, 544, 493, 510, 505, 493, 516, 522, 522, 534, 543, 529, 503, 496] payload_rows_per_dst=[5382, 5508, 5582, 5558, 5600, 5556, 5544, 5750] dup_rows_per_dst=[2490, 2568, 2634, 2748, 2496, 2540, 2628, 2952] combine_rows_per_src=[5382, 5508, 5582, 5558, 5600, 5556, 5544, 5750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38950 dup_rows=21100 dup_rows_remote=18478 dedup_ratio=0.321960 router_tokens_per_expert=[514, 504, 491, 523, 531, 496, 539, 535, 512, 508, 517, 482, 501, 533, 477, 524, 522, 525, 509, 527, 495, 504, 500, 517, 528, 539, 545, 518, 512, 498, 509, 479, 505, 516, 520, 531, 463, 515, 509, 533, 505, 507, 509, 512, 531, 507, 518, 502, 491, 529, 505, 532, 514, 496, 527, 511, 525, 498, 512, 538, 483, 485, 519, 506] payload_rows_per_dst=[5672, 5486, 5516, 5520, 5574, 5524, 5572, 5572] dup_rows_per_dst=[2594, 2622, 2682, 2736, 2610, 2658, 2638, 2560] combine_rows_per_src=[5672, 5486, 5516, 5520, 5574, 5524, 5572, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159293440 dispatch_meta_bytes=229080 combine_payload_bytes=159293440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159522520 ingress_bytes=184459264 link_bytes=184459264 token_bytes=318815960 weight_movement_bytes=25165824 total_nvlink_bytes=343981784 weight_to_token_ratio=0.078935 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403521868e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=229256 combine_payload_bytes=159268864 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159498120 ingress_bytes=209600512 link_bytes=209600512 token_bytes=318766984 weight_movement_bytes=50331648 total_nvlink_bytes=369098632 weight_to_token_ratio=0.157895 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594817716e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44504 payload_rows_remote=38862 dup_rows=21032 dup_rows_remote=18268 dedup_ratio=0.320923 router_tokens_per_expert=[514, 529, 478, 506, 528, 487, 488, 518, 495, 540, 506, 496, 499, 546, 515, 512, 493, 516, 541, 498, 495, 511, 492, 549, 515, 544, 496, 484, 546, 521, 493, 510, 540, 495, 496, 502, 474, 481, 542, 481, 498, 539, 523, 501, 565, 521, 529, 527, 507, 528, 525, 482, 513, 513, 535, 503, 490, 540, 508, 533, 503, 504, 497, 512] payload_rows_per_dst=[5498, 5662, 5572, 5540, 5492, 5642, 5542, 5556] dup_rows_per_dst=[2598, 2556, 2618, 2678, 2530, 2764, 2670, 2618] combine_rows_per_src=[5498, 5662, 5572, 5540, 5492, 5642, 5542, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159391744 dispatch_meta_bytes=229432 combine_payload_bytes=159391744 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159621176 ingress_bytes=184557568 link_bytes=184557568 token_bytes=319012920 weight_movement_bytes=25165824 total_nvlink_bytes=344178744 weight_to_token_ratio=0.078887 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404269848e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158638080 dispatch_meta_bytes=227336 combine_payload_bytes=158638080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158865416 ingress_bytes=183803904 link_bytes=183803904 token_bytes=317503496 weight_movement_bytes=25165824 total_nvlink_bytes=342669320 weight_to_token_ratio=0.079262 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398535336e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159735808 dispatch_meta_bytes=229576 combine_payload_bytes=159735808 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159965384 ingress_bytes=210067456 link_bytes=210067456 token_bytes=319701192 weight_movement_bytes=50331648 total_nvlink_bytes=370032840 weight_to_token_ratio=0.157433 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598370621e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38872 dup_rows=21068 dup_rows_remote=18250 dedup_ratio=0.321472 router_tokens_per_expert=[507, 520, 526, 539, 506, 499, 538, 505, 547, 546, 505, 495, 549, 522, 486, 457, 525, 508, 515, 512, 477, 530, 508, 506, 474, 483, 478, 500, 531, 509, 472, 534, 475, 503, 509, 509, 536, 525, 523, 536, 569, 531, 490, 509, 482, 507, 500, 507, 514, 526, 544, 491, 507, 516, 513, 523, 527, 519, 496, 515, 492, 499, 495, 571] payload_rows_per_dst=[5596, 5500, 5620, 5420, 5736, 5454, 5594, 5548] dup_rows_per_dst=[2818, 2620, 2664, 2420, 2832, 2528, 2506, 2680] combine_rows_per_src=[5596, 5500, 5620, 5420, 5736, 5454, 5594, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159539200 dispatch_meta_bytes=229712 combine_payload_bytes=159539200 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159768912 ingress_bytes=235036672 link_bytes=235036672 token_bytes=319308112 weight_movement_bytes=75497472 total_nvlink_bytes=394805584 weight_to_token_ratio=0.236441 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788357504e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159178752 dispatch_meta_bytes=228520 combine_payload_bytes=159178752 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159407272 ingress_bytes=209510400 link_bytes=209510400 token_bytes=318586024 weight_movement_bytes=50331648 total_nvlink_bytes=368917672 weight_to_token_ratio=0.157984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594132068e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159219712 dispatch_meta_bytes=228488 combine_payload_bytes=159219712 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159448200 ingress_bytes=209551360 link_bytes=209551360 token_bytes=318667912 weight_movement_bytes=50331648 total_nvlink_bytes=368999560 weight_to_token_ratio=0.157944 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594443726e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44514 payload_rows_remote=38946 dup_rows=21022 dup_rows_remote=18482 dedup_ratio=0.320770 router_tokens_per_expert=[503, 499, 489, 474, 491, 504, 492, 494, 500, 498, 537, 521, 557, 504, 491, 473, 516, 489, 466, 533, 506, 514, 517, 513, 525, 521, 545, 519, 497, 489, 531, 548, 497, 507, 537, 502, 493, 524, 526, 531, 519, 486, 507, 525, 488, 531, 513, 495, 501, 519, 561, 519, 551, 514, 541, 522, 538, 507, 510, 547, 512, 514, 456, 519] payload_rows_per_dst=[5402, 5528, 5568, 5650, 5594, 5486, 5738, 5548] dup_rows_per_dst=[2490, 2634, 2540, 2700, 2640, 2642, 2718, 2658] combine_rows_per_src=[5402, 5528, 5568, 5650, 5594, 5486, 5738, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44562 payload_rows_remote=38932 dup_rows=20974 dup_rows_remote=18252 dedup_ratio=0.320038 router_tokens_per_expert=[536, 535, 540, 498, 508, 500, 501, 496, 499, 510, 522, 503, 481, 505, 530, 511, 498, 514, 465, 516, 519, 505, 526, 509, 530, 556, 521, 504, 504, 534, 545, 510, 516, 517, 522, 515, 515, 495, 541, 519, 496, 504, 471, 507, 471, 497, 521, 500, 523, 543, 519, 496, 498, 559, 521, 517, 477, 543, 527, 518, 489, 531, 471, 498] payload_rows_per_dst=[5628, 5506, 5532, 5666, 5586, 5460, 5630, 5554] dup_rows_per_dst=[2600, 2616, 2572, 2742, 2694, 2474, 2722, 2554] combine_rows_per_src=[5628, 5506, 5532, 5666, 5586, 5460, 5630, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44456 payload_rows_remote=38774 dup_rows=21080 dup_rows_remote=18120 dedup_ratio=0.321655 router_tokens_per_expert=[516, 545, 507, 513, 532, 500, 495, 504, 502, 532, 486, 508, 552, 491, 524, 473, 573, 514, 509, 488, 492, 516, 528, 514, 515, 487, 506, 534, 509, 486, 527, 472, 508, 499, 521, 530, 532, 485, 545, 526, 534, 500, 517, 539, 511, 471, 550, 471, 531, 547, 503, 521, 511, 528, 478, 527, 499, 478, 483, 511, 535, 523, 493, 511] payload_rows_per_dst=[5632, 5572, 5534, 5394, 5636, 5640, 5366, 5682] dup_rows_per_dst=[2592, 2564, 2734, 2360, 2656, 2864, 2350, 2960] combine_rows_per_src=[5632, 5572, 5534, 5394, 5636, 5640, 5366, 5682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44396 payload_rows_remote=38740 dup_rows=21140 dup_rows_remote=18422 dedup_ratio=0.322571 router_tokens_per_expert=[525, 519, 507, 487, 498, 480, 507, 504, 505, 531, 485, 511, 506, 488, 531, 491, 537, 486, 460, 524, 513, 534, 514, 514, 501, 531, 494, 539, 524, 530, 484, 522, 549, 509, 523, 544, 501, 530, 529, 502, 553, 509, 512, 491, 507, 492, 528, 496, 483, 516, 517, 550, 549, 490, 509, 485, 499, 504, 525, 503, 515, 517, 546, 503] payload_rows_per_dst=[5446, 5562, 5480, 5564, 5656, 5556, 5542, 5590] dup_rows_per_dst=[2608, 2534, 2684, 2686, 2718, 2620, 2656, 2634] combine_rows_per_src=[5446, 5562, 5480, 5564, 5656, 5556, 5542, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=38936 dup_rows=21002 dup_rows_remote=18388 dedup_ratio=0.320465 router_tokens_per_expert=[547, 516, 532, 496, 483, 515, 512, 517, 509, 508, 485, 527, 506, 519, 515, 536, 508, 473, 484, 527, 483, 512, 512, 518, 544, 504, 498, 501, 533, 510, 513, 503, 511, 492, 471, 492, 511, 534, 545, 483, 529, 540, 515, 498, 524, 512, 500, 530, 510, 507, 501, 537, 520, 520, 526, 525, 511, 510, 502, 539, 497, 521, 475, 534] payload_rows_per_dst=[5578, 5558, 5474, 5598, 5548, 5552, 5638, 5588] dup_rows_per_dst=[2658, 2652, 2560, 2614, 2530, 2744, 2654, 2590] combine_rows_per_src=[5578, 5558, 5474, 5598, 5548, 5552, 5638, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229712 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159752528 ingress_bytes=184688640 link_bytes=184688640 token_bytes=319275344 weight_movement_bytes=25165824 total_nvlink_bytes=344441168 weight_to_token_ratio=0.078822 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405267154e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44512 payload_rows_remote=38926 dup_rows=21024 dup_rows_remote=18290 dedup_ratio=0.320801 router_tokens_per_expert=[521, 491, 537, 516, 513, 530, 514, 522, 527, 494, 546, 521, 521, 517, 520, 514, 505, 515, 485, 539, 489, 477, 502, 508, 507, 470, 563, 526, 497, 543, 529, 463, 536, 492, 541, 492, 539, 489, 490, 481, 503, 510, 513, 516, 498, 462, 517, 519, 495, 507, 541, 481, 483, 548, 538, 513, 501, 539, 522, 510, 524, 537, 518, 491] payload_rows_per_dst=[5594, 5586, 5510, 5620, 5580, 5530, 5508, 5584] dup_rows_per_dst=[2694, 2734, 2530, 2576, 2540, 2546, 2704, 2700] combine_rows_per_src=[5594, 5586, 5510, 5620, 5580, 5530, 5508, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=38964 dup_rows=20916 dup_rows_remote=18370 dedup_ratio=0.319153 router_tokens_per_expert=[522, 539, 473, 524, 544, 514, 533, 503, 562, 489, 491, 505, 506, 498, 539, 502, 528, 513, 530, 551, 506, 536, 498, 508, 491, 490, 501, 514, 469, 511, 519, 508, 518, 515, 505, 501, 520, 502, 524, 504, 512, 500, 499, 542, 506, 528, 523, 491, 524, 538, 522, 505, 527, 523, 500, 545, 491, 496, 520, 523, 491, 483, 505, 468] payload_rows_per_dst=[5590, 5594, 5614, 5508, 5548, 5656, 5650, 5460] dup_rows_per_dst=[2714, 2590, 2726, 2498, 2630, 2546, 2718, 2494] combine_rows_per_src=[5590, 5594, 5614, 5508, 5548, 5656, 5650, 5460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159465472 dispatch_meta_bytes=228736 combine_payload_bytes=159465472 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159694208 ingress_bytes=260128768 link_bytes=260128768 token_bytes=319159680 weight_movement_bytes=100663296 total_nvlink_bytes=419822976 weight_to_token_ratio=0.315401 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.979279362e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158818304 dispatch_meta_bytes=227576 combine_payload_bytes=158818304 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159045880 ingress_bytes=183984128 link_bytes=183984128 token_bytes=317864184 weight_movement_bytes=25165824 total_nvlink_bytes=343030008 weight_to_token_ratio=0.079172 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399906632e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44340 payload_rows_remote=38736 dup_rows=21196 dup_rows_remote=18490 dedup_ratio=0.323425 router_tokens_per_expert=[509, 549, 460, 506, 542, 512, 500, 481, 506, 544, 554, 528, 520, 526, 513, 540, 497, 552, 549, 499, 523, 478, 442, 558, 537, 509, 487, 526, 585, 510, 524, 492, 535, 504, 524, 558, 514, 493, 502, 475, 533, 508, 522, 501, 507, 485, 512, 542, 525, 473, 505, 541, 482, 512, 492, 495, 535, 536, 478, 483, 469, 509, 465, 495] payload_rows_per_dst=[5604, 5600, 5630, 5600, 5490, 5544, 5450, 5422] dup_rows_per_dst=[2706, 2706, 2848, 2760, 2490, 2676, 2492, 2518] combine_rows_per_src=[5604, 5600, 5630, 5600, 5490, 5544, 5450, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158679040 dispatch_meta_bytes=228648 combine_payload_bytes=158679040 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158907688 ingress_bytes=209010688 link_bytes=209010688 token_bytes=317586728 weight_movement_bytes=50331648 total_nvlink_bytes=367918376 weight_to_token_ratio=0.158482 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590329837e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159481856 dispatch_meta_bytes=229296 combine_payload_bytes=159481856 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159711152 ingress_bytes=209813504 link_bytes=209813504 token_bytes=319193008 weight_movement_bytes=50331648 total_nvlink_bytes=369524656 weight_to_token_ratio=0.157684 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596438339e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159440896 dispatch_meta_bytes=228864 combine_payload_bytes=159440896 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159669760 ingress_bytes=209772544 link_bytes=209772544 token_bytes=319110656 weight_movement_bytes=50331648 total_nvlink_bytes=369442304 weight_to_token_ratio=0.157725 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596126681e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229336 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159825880 ingress_bytes=184762368 link_bytes=184762368 token_bytes=319422424 weight_movement_bytes=25165824 total_nvlink_bytes=344588248 weight_to_token_ratio=0.078785 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405828139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38904 dup_rows=21114 dup_rows_remote=18566 dedup_ratio=0.322174 router_tokens_per_expert=[493, 520, 512, 519, 522, 482, 485, 543, 505, 548, 482, 526, 521, 538, 489, 504, 510, 522, 523, 485, 480, 491, 507, 515, 522, 507, 521, 551, 505, 512, 536, 484, 524, 502, 535, 522, 501, 506, 517, 500, 501, 542, 482, 522, 508, 531, 511, 531, 477, 489, 499, 509, 499, 509, 526, 478, 521, 538, 498, 497, 565, 527, 528, 513] payload_rows_per_dst=[5558, 5514, 5518, 5528, 5602, 5652, 5514, 5536] dup_rows_per_dst=[2594, 2712, 2548, 2748, 2612, 2604, 2458, 2838] combine_rows_per_src=[5558, 5514, 5518, 5528, 5602, 5652, 5514, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158662656 dispatch_meta_bytes=228904 combine_payload_bytes=158662656 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158891560 ingress_bytes=183828480 link_bytes=183828480 token_bytes=317554216 weight_movement_bytes=25165824 total_nvlink_bytes=342720040 weight_to_token_ratio=0.079249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398722331e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44488 payload_rows_remote=38900 dup_rows=21048 dup_rows_remote=18472 dedup_ratio=0.321167 router_tokens_per_expert=[493, 486, 528, 507, 517, 538, 502, 528, 499, 508, 526, 508, 492, 483, 529, 496, 510, 513, 515, 513, 517, 511, 537, 510, 503, 477, 542, 541, 482, 530, 480, 500, 553, 515, 492, 535, 543, 516, 513, 519, 545, 494, 526, 519, 473, 539, 466, 490, 484, 540, 505, 525, 498, 507, 493, 530, 521, 493, 555, 494, 515, 506, 499, 544] payload_rows_per_dst=[5600, 5532, 5526, 5520, 5648, 5504, 5588, 5570] dup_rows_per_dst=[2598, 2550, 2726, 2590, 2724, 2600, 2576, 2684] combine_rows_per_src=[5600, 5532, 5526, 5520, 5648, 5504, 5588, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44566 payload_rows_remote=38876 dup_rows=20970 dup_rows_remote=18202 dedup_ratio=0.319977 router_tokens_per_expert=[501, 510, 515, 538, 530, 516, 498, 533, 511, 511, 519, 519, 519, 496, 525, 532, 511, 523, 509, 537, 527, 504, 514, 493, 547, 521, 492, 522, 523, 513, 539, 513, 502, 508, 510, 506, 499, 508, 511, 548, 533, 521, 498, 540, 487, 493, 489, 491, 508, 495, 495, 494, 464, 491, 459, 521, 481, 541, 492, 510, 538, 527, 539, 508] payload_rows_per_dst=[5522, 5592, 5540, 5662, 5614, 5386, 5560, 5690] dup_rows_per_dst=[2624, 2672, 2696, 2814, 2384, 2396, 2616, 2768] combine_rows_per_src=[5522, 5592, 5540, 5662, 5614, 5386, 5560, 5690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159350784 dispatch_meta_bytes=229880 combine_payload_bytes=159350784 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159580664 ingress_bytes=184516608 link_bytes=184516608 token_bytes=318931448 weight_movement_bytes=25165824 total_nvlink_bytes=344097272 weight_to_token_ratio=0.078907 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403958190e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44530 payload_rows_remote=38932 dup_rows=21006 dup_rows_remote=18430 dedup_ratio=0.320526 router_tokens_per_expert=[465, 498, 510, 545, 472, 557, 487, 489, 510, 488, 534, 517, 504, 529, 473, 481, 504, 521, 499, 525, 540, 513, 557, 530, 543, 543, 502, 516, 503, 514, 547, 490, 525, 507, 534, 503, 511, 472, 525, 510, 522, 493, 506, 546, 482, 496, 520, 514, 540, 522, 511, 519, 521, 501, 507, 514, 503, 509, 541, 489, 513, 478, 503, 525] payload_rows_per_dst=[5514, 5506, 5600, 5590, 5598, 5600, 5636, 5486] dup_rows_per_dst=[2532, 2566, 2778, 2726, 2576, 2558, 2634, 2636] combine_rows_per_src=[5514, 5506, 5600, 5590, 5598, 5600, 5636, 5486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44376 payload_rows_remote=38776 dup_rows=21160 dup_rows_remote=18448 dedup_ratio=0.322876 router_tokens_per_expert=[506, 501, 492, 539, 508, 488, 558, 508, 503, 538, 529, 489, 510, 453, 492, 509, 494, 512, 498, 528, 500, 494, 505, 465, 498, 526, 503, 529, 505, 553, 523, 551, 540, 521, 480, 511, 503, 495, 535, 513, 516, 542, 469, 531, 515, 536, 504, 543, 499, 497, 524, 514, 522, 495, 516, 543, 522, 495, 522, 489, 519, 505, 530, 515] payload_rows_per_dst=[5540, 5550, 5498, 5602, 5542, 5600, 5544, 5500] dup_rows_per_dst=[2660, 2496, 2494, 2774, 2654, 2712, 2676, 2694] combine_rows_per_src=[5540, 5550, 5498, 5602, 5542, 5600, 5544, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44580 payload_rows_remote=38936 dup_rows=20956 dup_rows_remote=18350 dedup_ratio=0.319763 router_tokens_per_expert=[512, 558, 526, 542, 517, 521, 484, 515, 502, 515, 491, 495, 523, 475, 460, 503, 514, 499, 529, 514, 524, 534, 516, 548, 529, 525, 522, 493, 510, 524, 495, 527, 530, 527, 492, 494, 483, 551, 503, 505, 512, 536, 544, 512, 498, 572, 518, 493, 545, 478, 496, 508, 500, 517, 508, 495, 471, 503, 491, 492, 510, 522, 524, 496] payload_rows_per_dst=[5564, 5512, 5648, 5644, 5536, 5658, 5546, 5472] dup_rows_per_dst=[2786, 2416, 2708, 2606, 2634, 2712, 2548, 2546] combine_rows_per_src=[5564, 5512, 5648, 5644, 5536, 5658, 5546, 5472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=38968 dup_rows=20916 dup_rows_remote=18432 dedup_ratio=0.319153 router_tokens_per_expert=[536, 488, 509, 487, 528, 533, 509, 540, 506, 495, 482, 520, 518, 496, 547, 504, 506, 517, 499, 524, 473, 546, 530, 495, 532, 521, 490, 532, 521, 511, 540, 485, 505, 520, 519, 520, 500, 515, 527, 488, 495, 505, 542, 533, 511, 520, 497, 520, 534, 516, 509, 527, 482, 545, 516, 528, 519, 512, 509, 500, 457, 491, 464, 522] payload_rows_per_dst=[5612, 5652, 5492, 5552, 5556, 5624, 5618, 5514] dup_rows_per_dst=[2648, 2484, 2688, 2712, 2632, 2622, 2696, 2434] combine_rows_per_src=[5612, 5652, 5492, 5552, 5556, 5624, 5618, 5514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=229488 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159563888 ingress_bytes=184500224 link_bytes=184500224 token_bytes=318898288 weight_movement_bytes=25165824 total_nvlink_bytes=344064112 weight_to_token_ratio=0.078915 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403833526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44642 payload_rows_remote=39140 dup_rows=20894 dup_rows_remote=18212 dedup_ratio=0.318817 router_tokens_per_expert=[524, 512, 498, 516, 529, 520, 494, 495, 490, 535, 515, 467, 519, 506, 470, 524, 504, 493, 482, 509, 540, 534, 464, 509, 507, 515, 528, 516, 512, 521, 506, 494, 509, 505, 541, 510, 529, 504, 519, 487, 524, 517, 530, 515, 523, 523, 518, 519, 523, 536, 510, 509, 520, 558, 515, 492, 518, 499, 541, 494, 496, 480, 536, 520] payload_rows_per_dst=[5502, 5842, 5586, 5314, 5598, 5638, 5634, 5528] dup_rows_per_dst=[2682, 2940, 2490, 2154, 2626, 2670, 2692, 2640] combine_rows_per_src=[5502, 5842, 5586, 5314, 5598, 5638, 5634, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159236096 dispatch_meta_bytes=228312 combine_payload_bytes=159236096 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159464408 ingress_bytes=184401920 link_bytes=184401920 token_bytes=318700504 weight_movement_bytes=25165824 total_nvlink_bytes=343866328 weight_to_token_ratio=0.078964 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403085547e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159465472 dispatch_meta_bytes=229448 combine_payload_bytes=159465472 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159694920 ingress_bytes=209797120 link_bytes=209797120 token_bytes=319160392 weight_movement_bytes=50331648 total_nvlink_bytes=369492040 weight_to_token_ratio=0.157700 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596313676e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159481856 dispatch_meta_bytes=229144 combine_payload_bytes=159481856 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159711000 ingress_bytes=209813504 link_bytes=209813504 token_bytes=319192856 weight_movement_bytes=50331648 total_nvlink_bytes=369524504 weight_to_token_ratio=0.157684 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596438339e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=228896 combine_payload_bytes=158826496 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159055392 ingress_bytes=259489792 link_bytes=259489792 token_bytes=317881888 weight_movement_bytes=100663296 total_nvlink_bytes=418545184 weight_to_token_ratio=0.316669 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.974417493e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44350 payload_rows_remote=38796 dup_rows=21186 dup_rows_remote=18468 dedup_ratio=0.323273 router_tokens_per_expert=[518, 496, 538, 515, 513, 523, 548, 499, 503, 484, 540, 501, 512, 501, 477, 478, 510, 507, 507, 502, 561, 526, 500, 523, 503, 494, 493, 529, 494, 517, 514, 505, 484, 531, 567, 514, 481, 557, 553, 507, 491, 547, 499, 555, 506, 502, 521, 487, 518, 527, 524, 473, 533, 547, 521, 490, 480, 500, 523, 478, 492, 518, 498, 513] payload_rows_per_dst=[5588, 5480, 5554, 5514, 5672, 5502, 5586, 5454] dup_rows_per_dst=[2712, 2512, 2718, 2584, 2716, 2714, 2680, 2550] combine_rows_per_src=[5588, 5480, 5554, 5514, 5672, 5502, 5586, 5454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159612928 dispatch_meta_bytes=229600 combine_payload_bytes=159612928 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159842528 ingress_bytes=184778752 link_bytes=184778752 token_bytes=319455456 weight_movement_bytes=25165824 total_nvlink_bytes=344621280 weight_to_token_ratio=0.078777 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405952803e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=160317440 dispatch_meta_bytes=229408 combine_payload_bytes=160317440 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160546848 ingress_bytes=210649088 link_bytes=210649088 token_bytes=320864288 weight_movement_bytes=50331648 total_nvlink_bytes=371195936 weight_to_token_ratio=0.156863 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602796168e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44402 payload_rows_remote=38900 dup_rows=21134 dup_rows_remote=18518 dedup_ratio=0.322479 router_tokens_per_expert=[534, 487, 473, 530, 497, 528, 517, 515, 511, 456, 498, 515, 482, 544, 535, 530, 532, 508, 526, 503, 487, 517, 513, 469, 501, 500, 515, 518, 518, 493, 515, 520, 519, 501, 488, 529, 525, 505, 515, 535, 501, 533, 547, 534, 496, 531, 548, 501, 504, 519, 529, 518, 520, 509, 470, 490, 511, 528, 521, 510, 511, 494, 514, 525] payload_rows_per_dst=[5594, 5502, 5514, 5540, 5606, 5644, 5502, 5500] dup_rows_per_dst=[2568, 2640, 2596, 2620, 2628, 2738, 2616, 2728] combine_rows_per_src=[5594, 5502, 5514, 5540, 5606, 5644, 5502, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158908416 dispatch_meta_bytes=229056 combine_payload_bytes=158908416 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159137472 ingress_bytes=184074240 link_bytes=184074240 token_bytes=318045888 weight_movement_bytes=25165824 total_nvlink_bytes=343211712 weight_to_token_ratio=0.079126 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400592280e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44668 payload_rows_remote=39004 dup_rows=20868 dup_rows_remote=18214 dedup_ratio=0.318420 router_tokens_per_expert=[528, 542, 516, 522, 537, 485, 517, 559, 479, 490, 501, 509, 508, 503, 490, 502, 519, 506, 530, 536, 509, 511, 502, 496, 515, 493, 518, 489, 537, 502, 523, 494, 559, 493, 527, 503, 526, 521, 508, 516, 500, 503, 523, 508, 484, 528, 534, 484, 499, 507, 498, 532, 492, 501, 487, 536, 522, 508, 531, 537, 507, 512, 498, 516] payload_rows_per_dst=[5638, 5466, 5642, 5606, 5648, 5488, 5516, 5664] dup_rows_per_dst=[2636, 2498, 2714, 2530, 2658, 2640, 2538, 2654] combine_rows_per_src=[5638, 5466, 5642, 5606, 5648, 5488, 5516, 5664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44656 payload_rows_remote=39014 dup_rows=20880 dup_rows_remote=18186 dedup_ratio=0.318604 router_tokens_per_expert=[495, 515, 511, 507, 549, 516, 507, 531, 481, 538, 485, 476, 510, 556, 504, 486, 482, 515, 481, 495, 530, 514, 512, 516, 524, 508, 514, 553, 511, 538, 531, 556, 540, 528, 499, 526, 499, 526, 507, 543, 483, 521, 508, 525, 492, 450, 515, 505, 510, 539, 502, 505, 512, 489, 505, 534, 506, 541, 518, 517, 512, 491, 493, 480] payload_rows_per_dst=[5588, 5536, 5538, 5720, 5642, 5486, 5554, 5592] dup_rows_per_dst=[2674, 2536, 2552, 2750, 2694, 2512, 2638, 2524] combine_rows_per_src=[5588, 5536, 5538, 5720, 5642, 5486, 5554, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44338 payload_rows_remote=38886 dup_rows=21198 dup_rows_remote=18656 dedup_ratio=0.323456 router_tokens_per_expert=[498, 483, 520, 519, 496, 519, 497, 493, 495, 493, 509, 518, 485, 512, 506, 549, 505, 523, 538, 523, 495, 516, 508, 526, 464, 494, 497, 479, 529, 502, 513, 519, 517, 497, 508, 535, 537, 526, 503, 525, 542, 558, 472, 524, 486, 540, 487, 540, 546, 522, 457, 488, 538, 517, 529, 531, 494, 506, 523, 507, 547, 533, 516, 494] payload_rows_per_dst=[5520, 5500, 5574, 5452, 5602, 5562, 5556, 5572] dup_rows_per_dst=[2530, 2634, 2694, 2542, 2694, 2736, 2700, 2668] combine_rows_per_src=[5520, 5500, 5574, 5452, 5602, 5562, 5556, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44484 payload_rows_remote=38856 dup_rows=21052 dup_rows_remote=18456 dedup_ratio=0.321228 router_tokens_per_expert=[547, 515, 518, 508, 461, 515, 521, 509, 489, 522, 498, 519, 484, 501, 480, 505, 509, 499, 527, 506, 491, 518, 504, 521, 520, 515, 510, 520, 529, 511, 521, 520, 527, 508, 530, 445, 538, 499, 506, 514, 486, 530, 529, 522, 473, 512, 558, 502, 513, 533, 496, 503, 496, 518, 534, 526, 531, 517, 562, 532, 507, 491, 529, 488] payload_rows_per_dst=[5518, 5456, 5448, 5630, 5568, 5628, 5608, 5628] dup_rows_per_dst=[2670, 2540, 2702, 2662, 2566, 2596, 2630, 2686] combine_rows_per_src=[5518, 5456, 5448, 5630, 5568, 5628, 5608, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=229672 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159564072 ingress_bytes=259997696 link_bytes=259997696 token_bytes=318898472 weight_movement_bytes=100663296 total_nvlink_bytes=419561768 weight_to_token_ratio=0.315659 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978282056e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38886 dup_rows=21026 dup_rows_remote=18256 dedup_ratio=0.320831 router_tokens_per_expert=[500, 488, 506, 513, 506, 488, 528, 542, 548, 521, 501, 548, 509, 498, 523, 549, 530, 546, 490, 541, 481, 546, 495, 501, 535, 513, 551, 504, 495, 482, 512, 494, 508, 487, 498, 474, 477, 526, 468, 530, 475, 525, 511, 508, 508, 516, 527, 498, 524, 534, 524, 506, 493, 506, 546, 523, 496, 497, 495, 524, 511, 538, 489, 542] payload_rows_per_dst=[5554, 5624, 5608, 5524, 5412, 5600, 5642, 5546] dup_rows_per_dst=[2588, 2770, 2652, 2648, 2524, 2536, 2670, 2638] combine_rows_per_src=[5554, 5624, 5608, 5524, 5412, 5600, 5642, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159760384 dispatch_meta_bytes=228872 combine_payload_bytes=159760384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159989256 ingress_bytes=184926208 link_bytes=184926208 token_bytes=319749640 weight_movement_bytes=25165824 total_nvlink_bytes=344915464 weight_to_token_ratio=0.078705 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407074772e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44384 payload_rows_remote=38830 dup_rows=21152 dup_rows_remote=18548 dedup_ratio=0.322754 router_tokens_per_expert=[507, 481, 505, 498, 511, 511, 508, 546, 543, 521, 520, 515, 494, 528, 486, 506, 515, 516, 532, 490, 493, 489, 509, 471, 512, 504, 513, 519, 534, 477, 513, 535, 540, 483, 508, 527, 536, 532, 578, 477, 522, 538, 538, 488, 475, 507, 520, 494, 507, 523, 490, 528, 522, 577, 516, 474, 501, 525, 495, 484, 533, 485, 513, 530] payload_rows_per_dst=[5554, 5768, 5490, 5594, 5414, 5492, 5536, 5536] dup_rows_per_dst=[2604, 3074, 2540, 2620, 2492, 2662, 2564, 2596] combine_rows_per_src=[5554, 5768, 5490, 5594, 5414, 5492, 5536, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159801344 dispatch_meta_bytes=228800 combine_payload_bytes=159801344 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160030144 ingress_bytes=210132992 link_bytes=210132992 token_bytes=319831488 weight_movement_bytes=50331648 total_nvlink_bytes=370163136 weight_to_token_ratio=0.157369 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598869274e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=230168 combine_payload_bytes=159277056 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159507224 ingress_bytes=184442880 link_bytes=184442880 token_bytes=318784280 weight_movement_bytes=25165824 total_nvlink_bytes=343950104 weight_to_token_ratio=0.078943 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403397205e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38902 dup_rows=21088 dup_rows_remote=18422 dedup_ratio=0.321777 router_tokens_per_expert=[499, 530, 548, 527, 537, 512, 477, 488, 518, 536, 529, 555, 487, 530, 479, 484, 491, 536, 524, 520, 509, 522, 509, 495, 534, 556, 538, 536, 478, 493, 494, 446, 526, 490, 546, 510, 527, 543, 572, 491, 514, 523, 522, 475, 495, 487, 501, 528, 488, 483, 516, 523, 499, 530, 483, 488, 505, 491, 510, 513, 550, 528, 514, 480] payload_rows_per_dst=[5520, 5604, 5546, 5520, 5636, 5530, 5534, 5558] dup_rows_per_dst=[2716, 2632, 2666, 2630, 2774, 2560, 2486, 2624] combine_rows_per_src=[5520, 5604, 5546, 5520, 5636, 5530, 5534, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229248 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159383424 ingress_bytes=209485824 link_bytes=209485824 token_bytes=318537600 weight_movement_bytes=50331648 total_nvlink_bytes=368869248 weight_to_token_ratio=0.158008 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593945073e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=228568 combine_payload_bytes=159277056 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159505624 ingress_bytes=184442880 link_bytes=184442880 token_bytes=318782680 weight_movement_bytes=25165824 total_nvlink_bytes=343948504 weight_to_token_ratio=0.078944 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403397205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159047680 dispatch_meta_bytes=229512 combine_payload_bytes=159047680 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159277192 ingress_bytes=209379328 link_bytes=209379328 token_bytes=318324872 weight_movement_bytes=50331648 total_nvlink_bytes=368656520 weight_to_token_ratio=0.158114 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593134762e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=229296 combine_payload_bytes=159342592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159571888 ingress_bytes=184508416 link_bytes=184508416 token_bytes=318914480 weight_movement_bytes=25165824 total_nvlink_bytes=344080304 weight_to_token_ratio=0.078911 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403895858e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44626 payload_rows_remote=39132 dup_rows=20910 dup_rows_remote=18342 dedup_ratio=0.319061 router_tokens_per_expert=[534, 521, 551, 522, 513, 534, 515, 510, 484, 494, 516, 535, 495, 509, 527, 527, 518, 522, 525, 485, 515, 523, 489, 534, 520, 535, 512, 488, 514, 522, 543, 531, 527, 489, 533, 497, 509, 508, 509, 473, 542, 485, 509, 493, 483, 528, 506, 525, 492, 494, 531, 489, 514, 525, 487, 499, 502, 530, 476, 488, 493, 504, 535, 530] payload_rows_per_dst=[5684, 5604, 5604, 5622, 5556, 5556, 5494, 5506] dup_rows_per_dst=[2716, 2570, 2618, 2708, 2534, 2586, 2568, 2610] combine_rows_per_src=[5684, 5604, 5604, 5622, 5556, 5556, 5494, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44460 payload_rows_remote=38766 dup_rows=21076 dup_rows_remote=18166 dedup_ratio=0.321594 router_tokens_per_expert=[524, 486, 554, 543, 515, 513, 530, 531, 516, 547, 498, 482, 540, 508, 483, 532, 464, 518, 502, 532, 547, 476, 503, 507, 494, 473, 508, 495, 494, 503, 509, 497, 505, 494, 529, 538, 511, 542, 556, 506, 518, 467, 485, 550, 509, 522, 527, 526, 473, 500, 526, 496, 517, 527, 493, 515, 524, 541, 505, 499, 507, 507, 513, 516] payload_rows_per_dst=[5422, 5602, 5634, 5416, 5532, 5664, 5496, 5694] dup_rows_per_dst=[2496, 2610, 2698, 2444, 2466, 2784, 2668, 2910] combine_rows_per_src=[5422, 5602, 5634, 5416, 5532, 5664, 5496, 5694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=38972 dup_rows=21012 dup_rows_remote=18306 dedup_ratio=0.320618 router_tokens_per_expert=[509, 502, 510, 543, 503, 530, 496, 484, 549, 502, 513, 461, 524, 485, 506, 500, 495, 555, 537, 515, 514, 498, 517, 531, 501, 489, 523, 531, 499, 495, 577, 514, 501, 507, 552, 529, 519, 516, 513, 466, 542, 522, 482, 502, 520, 535, 526, 489, 490, 493, 518, 532, 527, 508, 519, 510, 481, 500, 503, 516, 438, 522, 547, 535] payload_rows_per_dst=[5532, 5510, 5540, 5552, 5656, 5604, 5578, 5552] dup_rows_per_dst=[2622, 2570, 2784, 2706, 2550, 2632, 2616, 2532] combine_rows_per_src=[5532, 5510, 5540, 5552, 5656, 5604, 5578, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=38980 dup_rows=20988 dup_rows_remote=18390 dedup_ratio=0.320251 router_tokens_per_expert=[487, 496, 506, 506, 495, 494, 528, 524, 484, 501, 529, 526, 540, 534, 518, 495, 508, 508, 514, 516, 505, 502, 546, 509, 517, 506, 538, 527, 548, 503, 511, 520, 495, 520, 511, 511, 495, 519, 508, 524, 531, 494, 482, 495, 529, 547, 502, 518, 532, 509, 500, 512, 484, 515, 475, 522, 520, 501, 520, 500, 512, 526, 519, 499] payload_rows_per_dst=[5500, 5682, 5534, 5608, 5568, 5594, 5518, 5544] dup_rows_per_dst=[2572, 2572, 2682, 2732, 2598, 2602, 2580, 2650] combine_rows_per_src=[5500, 5682, 5534, 5608, 5568, 5594, 5518, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38848 dup_rows=21108 dup_rows_remote=18524 dedup_ratio=0.322083 router_tokens_per_expert=[497, 484, 506, 515, 501, 489, 549, 504, 527, 534, 486, 548, 521, 503, 516, 494, 520, 479, 503, 515, 482, 534, 498, 531, 516, 484, 489, 540, 519, 527, 493, 521, 564, 512, 495, 519, 550, 548, 514, 492, 495, 521, 546, 500, 495, 484, 514, 527, 497, 507, 516, 534, 505, 516, 522, 523, 501, 529, 515, 501, 472, 527, 485, 517] payload_rows_per_dst=[5518, 5542, 5478, 5586, 5654, 5580, 5604, 5466] dup_rows_per_dst=[2572, 2716, 2646, 2592, 2734, 2584, 2636, 2628] combine_rows_per_src=[5518, 5542, 5478, 5586, 5654, 5580, 5604, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160284672 dispatch_meta_bytes=229896 combine_payload_bytes=160284672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160514568 ingress_bytes=185450496 link_bytes=185450496 token_bytes=320799240 weight_movement_bytes=25165824 total_nvlink_bytes=345965064 weight_to_token_ratio=0.078447 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411063998e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158785536 dispatch_meta_bytes=227728 combine_payload_bytes=158785536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159013264 ingress_bytes=183951360 link_bytes=183951360 token_bytes=317798800 weight_movement_bytes=25165824 total_nvlink_bytes=342964624 weight_to_token_ratio=0.079188 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399657306e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=39116 dup_rows=21000 dup_rows_remote=18354 dedup_ratio=0.320435 router_tokens_per_expert=[553, 508, 513, 490, 514, 540, 508, 521, 479, 481, 517, 517, 486, 539, 489, 525, 487, 520, 491, 504, 568, 502, 509, 477, 512, 528, 460, 500, 501, 484, 491, 566, 517, 508, 508, 497, 509, 507, 491, 507, 533, 566, 495, 497, 480, 477, 479, 546, 568, 544, 539, 553, 505, 518, 528, 511, 513, 518, 549, 502, 501, 503, 518, 501] payload_rows_per_dst=[5718, 5420, 5520, 5458, 5556, 5558, 5722, 5584] dup_rows_per_dst=[2576, 2646, 2596, 2626, 2532, 2588, 2810, 2626] combine_rows_per_src=[5718, 5420, 5520, 5458, 5556, 5558, 5722, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44462 payload_rows_remote=38800 dup_rows=21074 dup_rows_remote=18364 dedup_ratio=0.321564 router_tokens_per_expert=[512, 496, 497, 536, 504, 545, 525, 511, 508, 527, 479, 513, 561, 525, 522, 490, 525, 455, 520, 493, 505, 506, 509, 482, 504, 529, 531, 519, 515, 512, 547, 514, 517, 527, 506, 516, 534, 550, 511, 494, 537, 520, 470, 466, 504, 513, 534, 513, 525, 535, 495, 524, 517, 531, 483, 503, 485, 495, 548, 494, 531, 491, 479, 503] payload_rows_per_dst=[5662, 5598, 5456, 5560, 5544, 5550, 5570, 5522] dup_rows_per_dst=[2710, 2658, 2534, 2776, 2646, 2564, 2656, 2530] combine_rows_per_src=[5662, 5598, 5456, 5560, 5544, 5550, 5570, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=229112 combine_payload_bytes=159629312 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159858424 ingress_bytes=235126784 link_bytes=235126784 token_bytes=319487736 weight_movement_bytes=75497472 total_nvlink_bytes=394985208 weight_to_token_ratio=0.236308 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789043152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44538 payload_rows_remote=38954 dup_rows=20998 dup_rows_remote=18470 dedup_ratio=0.320404 router_tokens_per_expert=[494, 527, 533, 496, 532, 523, 509, 530, 525, 539, 545, 519, 465, 528, 515, 507, 517, 514, 517, 485, 538, 499, 490, 496, 507, 552, 495, 464, 493, 530, 494, 530, 506, 484, 513, 537, 509, 541, 486, 518, 486, 521, 522, 542, 515, 498, 485, 523, 539, 498, 518, 506, 494, 496, 545, 520, 499, 508, 479, 487, 544, 514, 504, 523] payload_rows_per_dst=[5622, 5602, 5584, 5554, 5546, 5530, 5576, 5524] dup_rows_per_dst=[2666, 2684, 2528, 2576, 2642, 2654, 2656, 2592] combine_rows_per_src=[5622, 5602, 5584, 5554, 5546, 5530, 5576, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159662080 dispatch_meta_bytes=229480 combine_payload_bytes=159662080 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159891560 ingress_bytes=235159552 link_bytes=235159552 token_bytes=319553640 weight_movement_bytes=75497472 total_nvlink_bytes=395051112 weight_to_token_ratio=0.236259 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789292479e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159121408 dispatch_meta_bytes=229488 combine_payload_bytes=159121408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159350896 ingress_bytes=184287232 link_bytes=184287232 token_bytes=318472304 weight_movement_bytes=25165824 total_nvlink_bytes=343638128 weight_to_token_ratio=0.079020 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402212903e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160219136 dispatch_meta_bytes=229880 combine_payload_bytes=160219136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160449016 ingress_bytes=185384960 link_bytes=185384960 token_bytes=320668152 weight_movement_bytes=25165824 total_nvlink_bytes=345833976 weight_to_token_ratio=0.078479 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410565345e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158924800 dispatch_meta_bytes=228656 combine_payload_bytes=158924800 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159153456 ingress_bytes=234422272 link_bytes=234422272 token_bytes=318078256 weight_movement_bytes=75497472 total_nvlink_bytes=393575728 weight_to_token_ratio=0.237355 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783682630e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=229696 combine_payload_bytes=159555584 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159785280 ingress_bytes=209887232 link_bytes=209887232 token_bytes=319340864 weight_movement_bytes=50331648 total_nvlink_bytes=369672512 weight_to_token_ratio=0.157611 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596999324e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38884 dup_rows=21064 dup_rows_remote=18394 dedup_ratio=0.321411 router_tokens_per_expert=[522, 509, 498, 517, 509, 514, 505, 503, 509, 536, 492, 522, 495, 540, 499, 529, 518, 493, 517, 523, 503, 514, 542, 538, 519, 550, 500, 502, 536, 542, 513, 501, 506, 484, 504, 523, 492, 550, 476, 464, 510, 521, 542, 467, 521, 489, 486, 505, 467, 509, 511, 508, 534, 561, 539, 500, 535, 515, 499, 519, 527, 467, 532, 495] payload_rows_per_dst=[5600, 5580, 5562, 5618, 5506, 5480, 5588, 5538] dup_rows_per_dst=[2554, 2664, 2734, 2708, 2492, 2602, 2670, 2640] combine_rows_per_src=[5600, 5580, 5562, 5618, 5506, 5480, 5588, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44388 payload_rows_remote=38824 dup_rows=21148 dup_rows_remote=18526 dedup_ratio=0.322693 router_tokens_per_expert=[507, 517, 492, 530, 507, 497, 511, 555, 494, 506, 535, 466, 521, 487, 540, 518, 535, 492, 513, 533, 514, 462, 501, 563, 524, 526, 487, 478, 484, 515, 495, 495, 544, 541, 480, 491, 490, 562, 500, 517, 507, 502, 497, 491, 542, 532, 521, 546, 513, 524, 525, 511, 490, 516, 500, 533, 488, 496, 511, 497, 545, 480, 558, 518] payload_rows_per_dst=[5502, 5546, 5382, 5644, 5542, 5668, 5540, 5564] dup_rows_per_dst=[2586, 2588, 2362, 2990, 2708, 2608, 2684, 2622] combine_rows_per_src=[5502, 5546, 5382, 5644, 5542, 5668, 5540, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44280 payload_rows_remote=38820 dup_rows=21256 dup_rows_remote=18724 dedup_ratio=0.324341 router_tokens_per_expert=[496, 508, 502, 501, 510, 509, 501, 555, 477, 502, 540, 564, 490, 503, 506, 485, 534, 510, 533, 501, 502, 491, 530, 493, 495, 487, 512, 493, 473, 522, 531, 483, 503, 526, 523, 526, 529, 551, 536, 498, 536, 466, 520, 537, 519, 488, 533, 507, 532, 515, 506, 508, 525, 522, 517, 516, 508, 526, 482, 554, 491, 489, 534, 506] payload_rows_per_dst=[5454, 5508, 5480, 5460, 5676, 5576, 5614, 5512] dup_rows_per_dst=[2710, 2626, 2708, 2532, 2708, 2636, 2668, 2668] combine_rows_per_src=[5454, 5508, 5480, 5460, 5676, 5576, 5614, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38954 dup_rows=21058 dup_rows_remote=18514 dedup_ratio=0.321320 router_tokens_per_expert=[487, 521, 510, 479, 504, 520, 517, 563, 514, 520, 552, 545, 507, 506, 507, 504, 540, 516, 518, 523, 511, 480, 520, 494, 501, 503, 533, 518, 502, 493, 512, 490, 531, 478, 486, 504, 514, 495, 499, 527, 548, 537, 518, 492, 489, 490, 517, 497, 514, 500, 512, 469, 525, 565, 524, 479, 531, 526, 533, 514, 511, 505, 530, 498] payload_rows_per_dst=[5560, 5558, 5602, 5452, 5524, 5586, 5604, 5592] dup_rows_per_dst=[2642, 2752, 2602, 2652, 2544, 2590, 2572, 2704] combine_rows_per_src=[5560, 5558, 5602, 5452, 5524, 5586, 5604, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44480 payload_rows_remote=38858 dup_rows=21056 dup_rows_remote=18378 dedup_ratio=0.321289 router_tokens_per_expert=[472, 511, 500, 517, 521, 491, 492, 529, 507, 529, 521, 539, 505, 503, 488, 481, 510, 542, 508, 542, 541, 525, 539, 514, 534, 521, 488, 471, 509, 505, 517, 459, 501, 508, 542, 498, 486, 503, 477, 520, 499, 498, 522, 535, 511, 525, 550, 510, 527, 519, 514, 521, 485, 534, 500, 501, 540, 526, 527, 528, 509, 482, 506, 533] payload_rows_per_dst=[5502, 5550, 5636, 5486, 5510, 5622, 5558, 5616] dup_rows_per_dst=[2564, 2596, 2806, 2522, 2560, 2678, 2644, 2686] combine_rows_per_src=[5502, 5550, 5636, 5486, 5510, 5622, 5558, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=229112 combine_payload_bytes=159268864 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159497976 ingress_bytes=209600512 link_bytes=209600512 token_bytes=318766840 weight_movement_bytes=50331648 total_nvlink_bytes=369098488 weight_to_token_ratio=0.157895 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594817716e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44618 payload_rows_remote=39078 dup_rows=20918 dup_rows_remote=18322 dedup_ratio=0.319183 router_tokens_per_expert=[509, 536, 562, 535, 518, 516, 501, 489, 500, 500, 489, 517, 529, 517, 493, 523, 510, 498, 538, 513, 514, 513, 521, 500, 513, 498, 526, 520, 491, 499, 521, 508, 526, 525, 519, 530, 519, 541, 496, 480, 472, 503, 519, 489, 500, 496, 461, 493, 502, 504, 526, 501, 537, 516, 497, 534, 542, 497, 513, 534, 507, 511, 529, 532] payload_rows_per_dst=[5574, 5540, 5630, 5538, 5594, 5472, 5578, 5692] dup_rows_per_dst=[2758, 2596, 2584, 2614, 2678, 2394, 2656, 2638] combine_rows_per_src=[5574, 5540, 5630, 5538, 5594, 5472, 5578, 5692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159023104 dispatch_meta_bytes=229400 combine_payload_bytes=159023104 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159252504 ingress_bytes=234520576 link_bytes=234520576 token_bytes=318275608 weight_movement_bytes=75497472 total_nvlink_bytes=393773080 weight_to_token_ratio=0.237208 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784430610e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44354 payload_rows_remote=38746 dup_rows=21182 dup_rows_remote=18548 dedup_ratio=0.323212 router_tokens_per_expert=[494, 491, 544, 522, 494, 527, 502, 535, 504, 477, 502, 496, 513, 475, 559, 515, 536, 538, 507, 526, 500, 522, 507, 532, 465, 514, 520, 533, 525, 510, 485, 543, 502, 543, 552, 521, 507, 504, 522, 509, 462, 505, 501, 552, 514, 483, 483, 505, 557, 507, 500, 503, 524, 506, 503, 495, 499, 489, 527, 505, 519, 542, 514, 500] payload_rows_per_dst=[5608, 5502, 5642, 5524, 5608, 5656, 5394, 5420] dup_rows_per_dst=[2634, 2644, 2694, 2666, 2698, 2784, 2534, 2528] combine_rows_per_src=[5608, 5502, 5642, 5524, 5608, 5656, 5394, 5420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44518 payload_rows_remote=38932 dup_rows=21018 dup_rows_remote=18338 dedup_ratio=0.320709 router_tokens_per_expert=[490, 530, 493, 556, 500, 518, 527, 556, 471, 557, 500, 472, 499, 482, 492, 533, 536, 544, 491, 496, 494, 475, 556, 541, 525, 482, 500, 479, 521, 492, 543, 576, 541, 501, 478, 529, 505, 504, 508, 530, 432, 539, 501, 546, 540, 525, 509, 521, 509, 525, 483, 519, 520, 483, 521, 507, 495, 490, 522, 523, 531, 495, 504, 505] payload_rows_per_dst=[5618, 5426, 5586, 5568, 5586, 5576, 5570, 5588] dup_rows_per_dst=[2722, 2586, 2680, 2668, 2606, 2650, 2564, 2542] combine_rows_per_src=[5618, 5426, 5586, 5568, 5586, 5576, 5570, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=230176 combine_payload_bytes=159006720 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159236896 ingress_bytes=184172544 link_bytes=184172544 token_bytes=318243616 weight_movement_bytes=25165824 total_nvlink_bytes=343409440 weight_to_token_ratio=0.079077 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401340260e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=229872 combine_payload_bytes=159555584 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159785456 ingress_bytes=209887232 link_bytes=209887232 token_bytes=319341040 weight_movement_bytes=50331648 total_nvlink_bytes=369672688 weight_to_token_ratio=0.157611 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596999324e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159162368 dispatch_meta_bytes=228944 combine_payload_bytes=159162368 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159391312 ingress_bytes=184328192 link_bytes=184328192 token_bytes=318553680 weight_movement_bytes=25165824 total_nvlink_bytes=343719504 weight_to_token_ratio=0.079000 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402524562e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160063488 dispatch_meta_bytes=229600 combine_payload_bytes=160063488 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160293088 ingress_bytes=185229312 link_bytes=185229312 token_bytes=320356576 weight_movement_bytes=25165824 total_nvlink_bytes=345522400 weight_to_token_ratio=0.078556 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409381044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158703616 dispatch_meta_bytes=229176 combine_payload_bytes=158703616 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158932792 ingress_bytes=209035264 link_bytes=209035264 token_bytes=317636408 weight_movement_bytes=50331648 total_nvlink_bytes=367968056 weight_to_token_ratio=0.158457 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590516832e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159465472 dispatch_meta_bytes=229080 combine_payload_bytes=159465472 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159694552 ingress_bytes=209797120 link_bytes=209797120 token_bytes=319160024 weight_movement_bytes=50331648 total_nvlink_bytes=369491672 weight_to_token_ratio=0.157700 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596313676e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44492 payload_rows_remote=38936 dup_rows=21044 dup_rows_remote=18470 dedup_ratio=0.321106 router_tokens_per_expert=[500, 523, 539, 536, 510, 505, 520, 461, 508, 547, 474, 513, 486, 511, 504, 497, 529, 539, 550, 522, 527, 528, 486, 560, 482, 571, 484, 514, 507, 493, 490, 495, 478, 557, 532, 508, 509, 513, 484, 505, 544, 480, 505, 557, 507, 474, 491, 495, 522, 522, 482, 479, 511, 532, 523, 494, 530, 501, 532, 534, 530, 492, 523, 511] payload_rows_per_dst=[5518, 5476, 5726, 5526, 5568, 5482, 5556, 5640] dup_rows_per_dst=[2670, 2604, 2756, 2546, 2604, 2624, 2574, 2666] combine_rows_per_src=[5518, 5476, 5726, 5526, 5568, 5482, 5556, 5640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44600 payload_rows_remote=38976 dup_rows=20936 dup_rows_remote=18196 dedup_ratio=0.319458 router_tokens_per_expert=[498, 507, 542, 533, 476, 522, 533, 528, 475, 497, 496, 480, 534, 514, 530, 528, 520, 525, 498, 522, 527, 512, 481, 504, 537, 563, 499, 475, 568, 496, 487, 508, 501, 497, 512, 502, 536, 522, 509, 529, 509, 506, 478, 495, 532, 496, 489, 516, 516, 530, 503, 499, 532, 520, 489, 521, 547, 519, 514, 508, 475, 500, 510, 541] payload_rows_per_dst=[5502, 5592, 5560, 5380, 5720, 5618, 5604, 5624] dup_rows_per_dst=[2560, 2516, 2618, 2512, 2712, 2662, 2616, 2740] combine_rows_per_src=[5502, 5592, 5560, 5380, 5720, 5618, 5604, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44332 payload_rows_remote=38716 dup_rows=21204 dup_rows_remote=18454 dedup_ratio=0.323547 router_tokens_per_expert=[500, 514, 538, 523, 515, 523, 524, 466, 519, 494, 510, 534, 493, 494, 526, 497, 504, 541, 503, 474, 531, 541, 516, 510, 507, 524, 509, 528, 508, 547, 514, 546, 493, 510, 516, 534, 455, 513, 500, 558, 532, 520, 510, 518, 523, 524, 510, 524, 515, 483, 484, 501, 491, 493, 466, 496, 540, 519, 529, 526, 521, 486, 514, 491] payload_rows_per_dst=[5552, 5538, 5528, 5616, 5546, 5540, 5428, 5584] dup_rows_per_dst=[2654, 2596, 2712, 2750, 2612, 2782, 2430, 2668] combine_rows_per_src=[5552, 5538, 5528, 5616, 5546, 5540, 5428, 5584] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44578 payload_rows_remote=38960 dup_rows=20958 dup_rows_remote=18332 dedup_ratio=0.319794 router_tokens_per_expert=[544, 505, 473, 516, 487, 520, 544, 525, 530, 517, 517, 480, 493, 480, 503, 483, 532, 536, 489, 528, 513, 492, 491, 541, 510, 547, 507, 549, 484, 515, 517, 500, 522, 502, 496, 492, 523, 525, 561, 501, 489, 520, 530, 508, 523, 516, 513, 502, 503, 503, 513, 507, 497, 477, 516, 502, 511, 480, 509, 550, 521, 517, 539, 532] payload_rows_per_dst=[5584, 5514, 5586, 5606, 5618, 5586, 5496, 5588] dup_rows_per_dst=[2644, 2492, 2658, 2652, 2626, 2616, 2540, 2730] combine_rows_per_src=[5584, 5514, 5586, 5606, 5618, 5586, 5496, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38940 dup_rows=21036 dup_rows_remote=18344 dedup_ratio=0.320984 router_tokens_per_expert=[487, 532, 491, 505, 494, 496, 533, 508, 502, 510, 520, 495, 528, 480, 532, 514, 498, 499, 507, 543, 491, 487, 508, 481, 517, 531, 484, 503, 535, 517, 516, 524, 503, 507, 525, 522, 504, 515, 529, 514, 526, 514, 567, 505, 514, 534, 475, 491, 537, 486, 539, 539, 543, 540, 540, 527, 516, 468, 518, 502, 487, 507, 480, 526] payload_rows_per_dst=[5548, 5554, 5464, 5560, 5624, 5560, 5686, 5504] dup_rows_per_dst=[2544, 2608, 2564, 2694, 2614, 2692, 2816, 2504] combine_rows_per_src=[5548, 5554, 5464, 5560, 5624, 5560, 5686, 5504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44632 payload_rows_remote=38918 dup_rows=20904 dup_rows_remote=18150 dedup_ratio=0.318970 router_tokens_per_expert=[527, 533, 508, 521, 502, 515, 534, 463, 549, 543, 517, 527, 499, 527, 539, 533, 498, 527, 526, 528, 518, 514, 540, 478, 498, 512, 534, 549, 505, 505, 500, 533, 472, 505, 506, 499, 497, 500, 497, 481, 512, 491, 483, 517, 508, 562, 488, 490, 531, 516, 545, 538, 498, 504, 492, 562, 472, 478, 470, 492, 521, 486, 530, 523] payload_rows_per_dst=[5614, 5714, 5574, 5564, 5430, 5604, 5630, 5502] dup_rows_per_dst=[2592, 2754, 2684, 2708, 2484, 2498, 2742, 2442] combine_rows_per_src=[5614, 5714, 5574, 5564, 5430, 5604, 5630, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159481856 dispatch_meta_bytes=229624 combine_payload_bytes=159481856 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159711480 ingress_bytes=234979328 link_bytes=234979328 token_bytes=319193336 weight_movement_bytes=75497472 total_nvlink_bytes=394690808 weight_to_token_ratio=0.236526 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787921182e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=228688 combine_payload_bytes=159645696 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159874384 ingress_bytes=184811520 link_bytes=184811520 token_bytes=319520080 weight_movement_bytes=25165824 total_nvlink_bytes=344685904 weight_to_token_ratio=0.078761 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406202129e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44324 payload_rows_remote=38756 dup_rows=21212 dup_rows_remote=18566 dedup_ratio=0.323669 router_tokens_per_expert=[520, 476, 537, 538, 542, 467, 515, 483, 510, 527, 506, 500, 497, 498, 511, 514, 500, 525, 524, 512, 523, 503, 496, 530, 517, 529, 508, 493, 502, 499, 524, 515, 541, 507, 500, 522, 498, 504, 520, 501, 525, 475, 533, 500, 538, 475, 481, 496, 559, 498, 537, 544, 525, 525, 494, 522, 502, 526, 505, 492, 494, 491, 566, 531] payload_rows_per_dst=[5568, 5686, 5374, 5542, 5540, 5446, 5606, 5562] dup_rows_per_dst=[2646, 2880, 2466, 2644, 2646, 2600, 2678, 2652] combine_rows_per_src=[5568, 5686, 5374, 5542, 5540, 5446, 5606, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44374 payload_rows_remote=38794 dup_rows=21162 dup_rows_remote=18486 dedup_ratio=0.322906 router_tokens_per_expert=[555, 511, 478, 509, 522, 523, 493, 440, 518, 458, 536, 488, 519, 508, 530, 520, 495, 537, 499, 531, 528, 483, 527, 528, 521, 510, 503, 513, 504, 528, 536, 531, 477, 485, 494, 509, 558, 510, 522, 464, 561, 552, 517, 519, 521, 522, 512, 515, 515, 491, 523, 464, 531, 515, 518, 505, 498, 518, 543, 519, 507, 485, 511, 505] payload_rows_per_dst=[5510, 5610, 5580, 5578, 5456, 5658, 5512, 5470] dup_rows_per_dst=[2552, 2544, 2676, 2714, 2582, 2780, 2612, 2702] combine_rows_per_src=[5510, 5610, 5580, 5578, 5456, 5658, 5512, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158580736 dispatch_meta_bytes=228680 combine_payload_bytes=158580736 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=158809416 ingress_bytes=234078208 link_bytes=234078208 token_bytes=317390152 weight_movement_bytes=75497472 total_nvlink_bytes=392887624 weight_to_token_ratio=0.237870 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.781064700e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159580160 dispatch_meta_bytes=229168 combine_payload_bytes=159580160 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159809328 ingress_bytes=184745984 link_bytes=184745984 token_bytes=319389488 weight_movement_bytes=25165824 total_nvlink_bytes=344555312 weight_to_token_ratio=0.078794 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405703476e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=229136 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159727376 ingress_bytes=184664064 link_bytes=184664064 token_bytes=319225616 weight_movement_bytes=25165824 total_nvlink_bytes=344391440 weight_to_token_ratio=0.078834 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405080160e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=228272 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159636400 ingress_bytes=184573952 link_bytes=184573952 token_bytes=319044528 weight_movement_bytes=25165824 total_nvlink_bytes=344210352 weight_to_token_ratio=0.078879 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404394511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158900224 dispatch_meta_bytes=229120 combine_payload_bytes=158900224 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159129344 ingress_bytes=234397696 link_bytes=234397696 token_bytes=318029568 weight_movement_bytes=75497472 total_nvlink_bytes=393527040 weight_to_token_ratio=0.237391 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783495635e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158744576 dispatch_meta_bytes=229288 combine_payload_bytes=158744576 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158973864 ingress_bytes=209076224 link_bytes=209076224 token_bytes=317718440 weight_movement_bytes=50331648 total_nvlink_bytes=368050088 weight_to_token_ratio=0.158416 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590828490e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44682 payload_rows_remote=39130 dup_rows=20854 dup_rows_remote=18204 dedup_ratio=0.318207 router_tokens_per_expert=[508, 548, 479, 521, 515, 473, 521, 517, 499, 532, 461, 505, 488, 521, 527, 482, 512, 532, 536, 512, 530, 519, 516, 523, 485, 534, 492, 499, 520, 500, 485, 513, 518, 482, 502, 527, 531, 500, 529, 529, 532, 554, 504, 495, 519, 533, 507, 487, 508, 544, 504, 499, 529, 516, 509, 492, 503, 508, 551, 514, 537, 495, 498, 507] payload_rows_per_dst=[5460, 5614, 5688, 5550, 5650, 5576, 5552, 5592] dup_rows_per_dst=[2704, 2416, 2672, 2506, 2586, 2686, 2650, 2634] combine_rows_per_src=[5460, 5614, 5688, 5550, 5650, 5576, 5552, 5592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44586 payload_rows_remote=38982 dup_rows=20950 dup_rows_remote=18352 dedup_ratio=0.319672 router_tokens_per_expert=[512, 523, 522, 508, 509, 466, 483, 515, 527, 497, 505, 522, 513, 512, 499, 511, 441, 520, 522, 481, 543, 554, 492, 525, 527, 497, 527, 519, 503, 506, 516, 537, 506, 493, 522, 486, 494, 510, 524, 504, 521, 550, 506, 509, 515, 513, 491, 530, 520, 514, 520, 490, 525, 510, 553, 533, 496, 541, 474, 510, 537, 484, 535, 518] payload_rows_per_dst=[5538, 5578, 5500, 5576, 5618, 5562, 5610, 5604] dup_rows_per_dst=[2538, 2594, 2382, 2676, 2734, 2558, 2870, 2598] combine_rows_per_src=[5538, 5578, 5500, 5576, 5618, 5562, 5610, 5604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44386 payload_rows_remote=38856 dup_rows=21150 dup_rows_remote=18558 dedup_ratio=0.322723 router_tokens_per_expert=[534, 529, 545, 521, 514, 514, 486, 521, 515, 508, 492, 517, 497, 513, 494, 506, 502, 545, 503, 528, 499, 514, 505, 515, 520, 535, 459, 544, 496, 535, 532, 440, 512, 524, 489, 501, 496, 541, 513, 516, 524, 523, 501, 517, 495, 528, 510, 525, 515, 508, 484, 516, 501, 519, 479, 533, 499, 502, 512, 545, 524, 508, 532, 498] payload_rows_per_dst=[5700, 5516, 5510, 5530, 5534, 5570, 5458, 5568] dup_rows_per_dst=[2628, 2568, 2712, 2592, 2650, 2676, 2652, 2672] combine_rows_per_src=[5700, 5516, 5510, 5530, 5534, 5570, 5458, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44550 payload_rows_remote=39030 dup_rows=20986 dup_rows_remote=18402 dedup_ratio=0.320221 router_tokens_per_expert=[524, 516, 523, 513, 483, 490, 493, 504, 547, 530, 492, 519, 503, 482, 532, 466, 500, 553, 514, 530, 501, 562, 498, 511, 505, 527, 490, 468, 500, 551, 521, 537, 502, 530, 516, 504, 529, 500, 488, 483, 501, 533, 508, 499, 463, 506, 503, 560, 486, 525, 522, 517, 520, 478, 513, 557, 524, 519, 529, 519, 502, 489, 536, 522] payload_rows_per_dst=[5552, 5556, 5646, 5528, 5520, 5582, 5610, 5556] dup_rows_per_dst=[2540, 2586, 2692, 2670, 2584, 2564, 2626, 2724] combine_rows_per_src=[5552, 5556, 5646, 5528, 5520, 5582, 5610, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38898 dup_rows=21026 dup_rows_remote=18306 dedup_ratio=0.320831 router_tokens_per_expert=[511, 504, 507, 542, 503, 471, 532, 530, 500, 489, 505, 490, 493, 498, 512, 524, 512, 519, 496, 568, 523, 489, 504, 521, 525, 543, 490, 499, 502, 539, 506, 513, 494, 552, 502, 531, 483, 512, 538, 522, 509, 517, 507, 530, 543, 526, 539, 495, 518, 493, 498, 551, 494, 532, 491, 517, 516, 476, 514, 496, 475, 506, 531, 500] payload_rows_per_dst=[5590, 5546, 5590, 5526, 5622, 5612, 5550, 5474] dup_rows_per_dst=[2610, 2476, 2674, 2708, 2646, 2720, 2638, 2554] combine_rows_per_src=[5590, 5546, 5590, 5526, 5622, 5612, 5550, 5474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44596 payload_rows_remote=39034 dup_rows=20940 dup_rows_remote=18278 dedup_ratio=0.319519 router_tokens_per_expert=[516, 493, 501, 533, 517, 509, 501, 510, 505, 497, 526, 520, 523, 515, 502, 524, 496, 527, 509, 500, 521, 537, 525, 491, 512, 523, 515, 501, 487, 521, 542, 495, 502, 511, 528, 532, 521, 465, 515, 538, 531, 500, 558, 519, 538, 516, 507, 519, 529, 492, 507, 485, 524, 486, 481, 484, 502, 526, 519, 507, 515, 507, 475, 535] payload_rows_per_dst=[5564, 5562, 5602, 5550, 5592, 5676, 5474, 5576] dup_rows_per_dst=[2596, 2662, 2610, 2642, 2632, 2700, 2502, 2596] combine_rows_per_src=[5564, 5562, 5602, 5550, 5592, 5676, 5474, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160276480 dispatch_meta_bytes=229336 combine_payload_bytes=160276480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160505816 ingress_bytes=185442304 link_bytes=185442304 token_bytes=320782296 weight_movement_bytes=25165824 total_nvlink_bytes=345948120 weight_to_token_ratio=0.078451 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411001667e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44658 payload_rows_remote=39004 dup_rows=20878 dup_rows_remote=18174 dedup_ratio=0.318573 router_tokens_per_expert=[471, 499, 520, 497, 506, 489, 515, 472, 511, 472, 540, 498, 540, 513, 499, 532, 509, 504, 518, 503, 578, 533, 519, 515, 538, 513, 513, 469, 515, 515, 490, 510, 506, 534, 536, 514, 504, 505, 526, 514, 519, 517, 520, 526, 513, 504, 492, 541, 475, 517, 507, 510, 508, 535, 513, 476, 519, 485, 506, 545, 504, 505, 520, 556] payload_rows_per_dst=[5486, 5608, 5654, 5510, 5582, 5592, 5566, 5660] dup_rows_per_dst=[2452, 2602, 2704, 2616, 2696, 2672, 2516, 2620] combine_rows_per_src=[5486, 5608, 5654, 5510, 5582, 5592, 5566, 5660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229336 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159899608 ingress_bytes=210001920 link_bytes=210001920 token_bytes=319569880 weight_movement_bytes=50331648 total_nvlink_bytes=369901528 weight_to_token_ratio=0.157498 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597871967e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229656 combine_payload_bytes=159154176 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159383832 ingress_bytes=209485824 link_bytes=209485824 token_bytes=318538008 weight_movement_bytes=50331648 total_nvlink_bytes=368869656 weight_to_token_ratio=0.158008 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593945073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44442 payload_rows_remote=38742 dup_rows=21094 dup_rows_remote=18340 dedup_ratio=0.321869 router_tokens_per_expert=[526, 549, 510, 506, 537, 504, 489, 493, 510, 502, 526, 536, 480, 500, 514, 551, 491, 491, 514, 504, 552, 502, 526, 486, 509, 500, 518, 527, 534, 537, 491, 527, 500, 537, 495, 496, 505, 497, 517, 518, 506, 484, 512, 577, 479, 496, 479, 491, 572, 522, 509, 520, 502, 459, 501, 525, 544, 520, 521, 493, 526, 515, 517, 491] payload_rows_per_dst=[5700, 5752, 5468, 5556, 5512, 5286, 5572, 5596] dup_rows_per_dst=[2754, 2956, 2534, 2634, 2618, 2330, 2648, 2620] combine_rows_per_src=[5700, 5752, 5468, 5556, 5512, 5286, 5572, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=229728 combine_payload_bytes=159866880 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160096608 ingress_bytes=185032704 link_bytes=185032704 token_bytes=319963488 weight_movement_bytes=25165824 total_nvlink_bytes=345129312 weight_to_token_ratio=0.078652 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407885084e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159326208 dispatch_meta_bytes=228816 combine_payload_bytes=159326208 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159555024 ingress_bytes=234823680 link_bytes=234823680 token_bytes=318881232 weight_movement_bytes=75497472 total_nvlink_bytes=394378704 weight_to_token_ratio=0.236757 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786736881e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=229248 combine_payload_bytes=159883264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160112512 ingress_bytes=185049088 link_bytes=185049088 token_bytes=319995776 weight_movement_bytes=25165824 total_nvlink_bytes=345161600 weight_to_token_ratio=0.078644 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408009747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159760384 dispatch_meta_bytes=228712 combine_payload_bytes=159760384 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159989096 ingress_bytes=210092032 link_bytes=210092032 token_bytes=319749480 weight_movement_bytes=50331648 total_nvlink_bytes=370081128 weight_to_token_ratio=0.157410 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598557616e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158687232 dispatch_meta_bytes=228328 combine_payload_bytes=158687232 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158915560 ingress_bytes=183853056 link_bytes=183853056 token_bytes=317602792 weight_movement_bytes=25165824 total_nvlink_bytes=342768616 weight_to_token_ratio=0.079237 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398909326e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44472 payload_rows_remote=38910 dup_rows=21064 dup_rows_remote=18412 dedup_ratio=0.321411 router_tokens_per_expert=[517, 522, 537, 526, 534, 532, 528, 526, 494, 568, 524, 514, 484, 498, 503, 526, 489, 509, 451, 493, 470, 535, 457, 559, 493, 510, 514, 476, 479, 524, 517, 541, 511, 530, 522, 508, 506, 502, 493, 487, 479, 532, 539, 519, 481, 526, 513, 554, 534, 503, 527, 503, 514, 504, 512, 510, 536, 492, 509, 553, 507, 516, 502, 494] payload_rows_per_dst=[5654, 5626, 5424, 5538, 5514, 5566, 5562, 5588] dup_rows_per_dst=[2790, 2596, 2502, 2570, 2604, 2720, 2652, 2630] combine_rows_per_src=[5654, 5626, 5424, 5538, 5514, 5566, 5562, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44450 payload_rows_remote=38894 dup_rows=21086 dup_rows_remote=18454 dedup_ratio=0.321747 router_tokens_per_expert=[505, 508, 508, 486, 514, 526, 545, 534, 504, 531, 495, 540, 507, 506, 494, 481, 514, 497, 521, 501, 548, 531, 533, 526, 534, 533, 524, 512, 530, 514, 535, 485, 496, 509, 550, 503, 506, 470, 503, 484, 506, 535, 496, 498, 523, 527, 524, 528, 508, 464, 504, 486, 501, 520, 529, 485, 508, 497, 530, 523, 506, 518, 508, 501] payload_rows_per_dst=[5434, 5300, 5454, 5622, 5788, 5604, 5692, 5556] dup_rows_per_dst=[2564, 2102, 2492, 2712, 2968, 2716, 2900, 2632] combine_rows_per_src=[5434, 5300, 5454, 5622, 5788, 5604, 5692, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38884 dup_rows=21098 dup_rows_remote=18392 dedup_ratio=0.321930 router_tokens_per_expert=[501, 510, 528, 474, 510, 495, 512, 511, 533, 511, 519, 530, 530, 548, 530, 506, 511, 530, 537, 518, 520, 480, 512, 493, 534, 506, 521, 526, 533, 500, 493, 517, 472, 485, 534, 515, 502, 519, 483, 486, 528, 551, 495, 494, 527, 494, 470, 477, 519, 510, 521, 517, 518, 510, 513, 536, 497, 511, 534, 529, 529, 503, 531, 479] payload_rows_per_dst=[5490, 5722, 5580, 5554, 5484, 5460, 5584, 5564] dup_rows_per_dst=[2592, 2692, 2622, 2706, 2508, 2612, 2704, 2662] combine_rows_per_src=[5490, 5722, 5580, 5554, 5484, 5460, 5584, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44570 payload_rows_remote=39000 dup_rows=20966 dup_rows_remote=18386 dedup_ratio=0.319916 router_tokens_per_expert=[520, 544, 522, 541, 468, 488, 555, 544, 491, 550, 513, 503, 513, 497, 509, 547, 530, 517, 534, 528, 529, 516, 515, 508, 487, 487, 505, 498, 502, 512, 521, 516, 503, 500, 510, 522, 488, 489, 555, 508, 519, 509, 479, 493, 512, 494, 513, 505, 500, 508, 525, 502, 499, 490, 499, 498, 515, 520, 523, 486, 520, 532, 531, 511] payload_rows_per_dst=[5528, 5582, 5684, 5566, 5570, 5526, 5524, 5590] dup_rows_per_dst=[2836, 2664, 2670, 2490, 2580, 2522, 2518, 2686] combine_rows_per_src=[5528, 5582, 5684, 5566, 5570, 5526, 5524, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44266 payload_rows_remote=38746 dup_rows=21270 dup_rows_remote=18656 dedup_ratio=0.324554 router_tokens_per_expert=[496, 474, 516, 505, 527, 490, 510, 538, 521, 516, 562, 505, 551, 522, 531, 504, 515, 513, 500, 494, 514, 540, 496, 526, 511, 507, 533, 496, 516, 514, 531, 456, 493, 518, 508, 506, 520, 538, 466, 503, 511, 514, 505, 514, 519, 479, 493, 532, 531, 542, 511, 533, 503, 516, 532, 474, 510, 471, 512, 504, 517, 535, 484, 544] payload_rows_per_dst=[5436, 5618, 5490, 5528, 5500, 5520, 5646, 5528] dup_rows_per_dst=[2676, 2806, 2706, 2600, 2604, 2614, 2638, 2626] combine_rows_per_src=[5436, 5618, 5490, 5528, 5500, 5520, 5646, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159375360 dispatch_meta_bytes=229288 combine_payload_bytes=159375360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159604648 ingress_bytes=184541184 link_bytes=184541184 token_bytes=318980008 weight_movement_bytes=25165824 total_nvlink_bytes=344145832 weight_to_token_ratio=0.078895 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404145185e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44378 payload_rows_remote=38858 dup_rows=21158 dup_rows_remote=18556 dedup_ratio=0.322845 router_tokens_per_expert=[494, 500, 571, 521, 519, 592, 474, 517, 535, 546, 516, 479, 512, 498, 504, 471, 533, 522, 532, 526, 521, 469, 504, 587, 527, 520, 498, 541, 540, 462, 500, 518, 502, 510, 474, 519, 470, 495, 528, 497, 533, 516, 531, 537, 520, 511, 492, 502, 487, 480, 492, 487, 478, 500, 545, 504, 507, 476, 521, 527, 484, 521, 527, 546] payload_rows_per_dst=[5660, 5520, 5602, 5560, 5424, 5592, 5404, 5616] dup_rows_per_dst=[2716, 2602, 2786, 2652, 2566, 2692, 2542, 2602] combine_rows_per_src=[5660, 5520, 5602, 5560, 5424, 5592, 5404, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44604 payload_rows_remote=38916 dup_rows=20932 dup_rows_remote=18256 dedup_ratio=0.319397 router_tokens_per_expert=[490, 505, 455, 515, 546, 459, 502, 452, 487, 479, 504, 519, 517, 555, 491, 504, 547, 525, 500, 536, 518, 513, 523, 520, 501, 537, 559, 551, 507, 492, 557, 512, 506, 530, 514, 501, 495, 529, 481, 531, 505, 480, 525, 534, 541, 491, 522, 519, 522, 493, 550, 489, 505, 497, 529, 523, 497, 543, 498, 506, 492, 503, 547, 492] payload_rows_per_dst=[5454, 5512, 5688, 5698, 5552, 5592, 5538, 5570] dup_rows_per_dst=[2394, 2600, 2676, 2734, 2622, 2642, 2678, 2586] combine_rows_per_src=[5454, 5512, 5688, 5698, 5552, 5592, 5538, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=229392 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159539216 ingress_bytes=209641472 link_bytes=209641472 token_bytes=318849040 weight_movement_bytes=50331648 total_nvlink_bytes=369180688 weight_to_token_ratio=0.157854 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595129375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159268864 dispatch_meta_bytes=229104 combine_payload_bytes=159268864 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159497968 ingress_bytes=184434688 link_bytes=184434688 token_bytes=318766832 weight_movement_bytes=25165824 total_nvlink_bytes=343932656 weight_to_token_ratio=0.078947 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403334873e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44348 payload_rows_remote=38836 dup_rows=21188 dup_rows_remote=18478 dedup_ratio=0.323303 router_tokens_per_expert=[511, 537, 493, 539, 515, 500, 491, 525, 533, 517, 527, 488, 517, 503, 497, 515, 529, 504, 543, 521, 477, 494, 485, 530, 502, 516, 497, 528, 530, 528, 494, 479, 485, 476, 525, 532, 542, 507, 520, 487, 546, 494, 515, 536, 514, 512, 530, 499, 469, 528, 514, 502, 503, 522, 506, 500, 492, 528, 544, 511, 537, 511, 504, 512] payload_rows_per_dst=[5512, 5724, 5508, 5618, 5538, 5548, 5464, 5436] dup_rows_per_dst=[2710, 2772, 2658, 2694, 2610, 2580, 2624, 2540] combine_rows_per_src=[5512, 5724, 5508, 5618, 5538, 5548, 5464, 5436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159744000 dispatch_meta_bytes=229544 combine_payload_bytes=159744000 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159973544 ingress_bytes=184909824 link_bytes=184909824 token_bytes=319717544 weight_movement_bytes=25165824 total_nvlink_bytes=344883368 weight_to_token_ratio=0.078713 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406950109e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158703616 dispatch_meta_bytes=229608 combine_payload_bytes=158703616 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158933224 ingress_bytes=209035264 link_bytes=209035264 token_bytes=317636840 weight_movement_bytes=50331648 total_nvlink_bytes=367968488 weight_to_token_ratio=0.158457 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590516832e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159162368 dispatch_meta_bytes=229656 combine_payload_bytes=159162368 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159392024 ingress_bytes=209494016 link_bytes=209494016 token_bytes=318554392 weight_movement_bytes=50331648 total_nvlink_bytes=368886040 weight_to_token_ratio=0.158000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594007405e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159399936 dispatch_meta_bytes=228688 combine_payload_bytes=159399936 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159628624 ingress_bytes=234897408 link_bytes=234897408 token_bytes=319028560 weight_movement_bytes=75497472 total_nvlink_bytes=394526032 weight_to_token_ratio=0.236648 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787297866e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159072256 dispatch_meta_bytes=229256 combine_payload_bytes=159072256 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159301512 ingress_bytes=234569728 link_bytes=234569728 token_bytes=318373768 weight_movement_bytes=75497472 total_nvlink_bytes=393871240 weight_to_token_ratio=0.237135 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784804600e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38832 dup_rows=21142 dup_rows_remote=18392 dedup_ratio=0.322601 router_tokens_per_expert=[497, 536, 534, 493, 503, 518, 513, 551, 451, 503, 515, 494, 545, 485, 500, 541, 509, 518, 522, 522, 542, 511, 488, 508, 523, 502, 515, 517, 530, 515, 542, 475, 509, 522, 524, 506, 505, 510, 477, 464, 536, 504, 502, 513, 525, 498, 471, 523, 505, 547, 496, 500, 495, 534, 530, 549, 510, 518, 489, 535, 485, 551, 508, 509] payload_rows_per_dst=[5604, 5510, 5656, 5532, 5470, 5504, 5562, 5556] dup_rows_per_dst=[2686, 2558, 2584, 2706, 2564, 2640, 2750, 2654] combine_rows_per_src=[5604, 5510, 5656, 5532, 5470, 5504, 5562, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44656 payload_rows_remote=39092 dup_rows=20880 dup_rows_remote=18212 dedup_ratio=0.318604 router_tokens_per_expert=[541, 539, 498, 505, 504, 502, 522, 512, 508, 501, 482, 501, 526, 465, 499, 526, 508, 512, 493, 519, 499, 508, 461, 520, 481, 542, 531, 534, 506, 509, 509, 495, 521, 543, 487, 524, 488, 530, 522, 508, 481, 503, 471, 543, 548, 526, 529, 502, 524, 504, 565, 523, 531, 516, 506, 513, 546, 497, 507, 500, 533, 508, 525, 486] payload_rows_per_dst=[5620, 5452, 5490, 5656, 5640, 5540, 5694, 5564] dup_rows_per_dst=[2626, 2564, 2362, 2718, 2606, 2634, 2702, 2668] combine_rows_per_src=[5620, 5452, 5490, 5656, 5640, 5540, 5694, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44534 payload_rows_remote=39030 dup_rows=21002 dup_rows_remote=18444 dedup_ratio=0.320465 router_tokens_per_expert=[548, 534, 526, 498, 512, 539, 535, 520, 496, 489, 518, 490, 498, 490, 518, 486, 536, 534, 506, 521, 511, 513, 465, 525, 484, 478, 514, 517, 497, 509, 534, 498, 527, 502, 498, 541, 510, 492, 486, 531, 526, 487, 490, 512, 522, 496, 506, 535, 537, 510, 498, 528, 554, 494, 489, 532, 537, 515, 576, 485, 503, 491, 534, 485] payload_rows_per_dst=[5634, 5448, 5622, 5504, 5590, 5560, 5582, 5594] dup_rows_per_dst=[2790, 2522, 2600, 2558, 2584, 2588, 2702, 2658] combine_rows_per_src=[5634, 5448, 5622, 5504, 5590, 5560, 5582, 5594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=38882 dup_rows=21012 dup_rows_remote=18266 dedup_ratio=0.320618 router_tokens_per_expert=[447, 470, 531, 518, 530, 552, 501, 505, 487, 526, 511, 513, 532, 509, 491, 530, 524, 539, 499, 515, 531, 486, 519, 513, 538, 497, 557, 504, 498, 549, 524, 510, 483, 524, 502, 546, 564, 523, 537, 515, 543, 506, 468, 505, 503, 503, 517, 496, 503, 542, 498, 514, 490, 505, 496, 504, 500, 477, 519, 451, 503, 503, 543, 529] payload_rows_per_dst=[5584, 5520, 5560, 5676, 5642, 5536, 5514, 5492] dup_rows_per_dst=[2524, 2678, 2692, 2678, 2746, 2546, 2590, 2558] combine_rows_per_src=[5584, 5520, 5560, 5676, 5642, 5536, 5514, 5492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159055872 dispatch_meta_bytes=228896 combine_payload_bytes=159055872 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159284768 ingress_bytes=209387520 link_bytes=209387520 token_bytes=318340640 weight_movement_bytes=50331648 total_nvlink_bytes=368672288 weight_to_token_ratio=0.158106 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593197093e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44412 payload_rows_remote=38758 dup_rows=21124 dup_rows_remote=18448 dedup_ratio=0.322327 router_tokens_per_expert=[497, 530, 485, 530, 513, 492, 507, 522, 515, 503, 478, 541, 541, 515, 524, 525, 524, 532, 521, 501, 482, 536, 521, 522, 579, 498, 497, 503, 508, 463, 515, 503, 515, 476, 473, 478, 519, 496, 496, 517, 531, 518, 500, 533, 539, 498, 519, 527, 515, 498, 540, 515, 521, 491, 518, 549, 515, 476, 492, 518, 499, 534, 506, 523] payload_rows_per_dst=[5514, 5520, 5576, 5554, 5434, 5654, 5590, 5570] dup_rows_per_dst=[2638, 2764, 2702, 2578, 2506, 2676, 2704, 2556] combine_rows_per_src=[5514, 5520, 5576, 5554, 5434, 5654, 5590, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160120832 dispatch_meta_bytes=229216 combine_payload_bytes=160120832 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160350048 ingress_bytes=210452480 link_bytes=210452480 token_bytes=320470880 weight_movement_bytes=50331648 total_nvlink_bytes=370802528 weight_to_token_ratio=0.157055 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601300208e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44692 payload_rows_remote=39140 dup_rows=20844 dup_rows_remote=18218 dedup_ratio=0.318054 router_tokens_per_expert=[526, 490, 542, 492, 520, 502, 494, 514, 498, 479, 517, 506, 537, 504, 514, 534, 497, 528, 507, 512, 504, 551, 530, 510, 511, 499, 526, 523, 497, 532, 544, 509, 510, 539, 540, 497, 496, 516, 515, 504, 521, 485, 529, 511, 529, 510, 479, 499, 532, 511, 506, 525, 496, 465, 488, 506, 506, 496, 519, 507, 533, 516, 507, 526] payload_rows_per_dst=[5582, 5552, 5628, 5582, 5646, 5538, 5584, 5580] dup_rows_per_dst=[2578, 2626, 2650, 2700, 2588, 2588, 2474, 2640] combine_rows_per_src=[5582, 5552, 5628, 5582, 5646, 5538, 5584, 5580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39092 dup_rows=20934 dup_rows_remote=18320 dedup_ratio=0.319427 router_tokens_per_expert=[486, 483, 507, 515, 493, 540, 530, 475, 500, 518, 528, 533, 555, 524, 506, 517, 516, 481, 514, 511, 502, 549, 516, 473, 512, 515, 494, 479, 508, 527, 507, 510, 547, 517, 526, 531, 525, 474, 518, 480, 517, 511, 517, 491, 483, 530, 482, 546, 549, 505, 519, 527, 513, 485, 504, 523, 493, 499, 515, 511, 510, 532, 540, 524] payload_rows_per_dst=[5530, 5632, 5510, 5560, 5590, 5560, 5598, 5622] dup_rows_per_dst=[2528, 2730, 2614, 2544, 2646, 2594, 2652, 2626] combine_rows_per_src=[5530, 5632, 5510, 5560, 5590, 5560, 5598, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159866880 dispatch_meta_bytes=229896 combine_payload_bytes=159866880 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160096776 ingress_bytes=210198528 link_bytes=210198528 token_bytes=319963656 weight_movement_bytes=50331648 total_nvlink_bytes=370295304 weight_to_token_ratio=0.157304 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599367927e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44662 payload_rows_remote=39032 dup_rows=20874 dup_rows_remote=18196 dedup_ratio=0.318512 router_tokens_per_expert=[561, 486, 469, 469, 537, 537, 497, 490, 508, 507, 518, 489, 541, 523, 510, 511, 544, 482, 519, 526, 485, 480, 491, 493, 499, 498, 514, 489, 508, 517, 524, 530, 498, 511, 527, 517, 520, 491, 492, 499, 517, 543, 531, 520, 563, 508, 469, 503, 499, 521, 505, 497, 478, 530, 499, 525, 532, 533, 544, 563, 503, 527, 531, 520] payload_rows_per_dst=[5630, 5816, 5532, 5326, 5634, 5658, 5444, 5622] dup_rows_per_dst=[2678, 2894, 2518, 2336, 2712, 2640, 2428, 2668] combine_rows_per_src=[5630, 5816, 5532, 5326, 5634, 5658, 5444, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159260672 dispatch_meta_bytes=228592 combine_payload_bytes=159260672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159489264 ingress_bytes=184426496 link_bytes=184426496 token_bytes=318749936 weight_movement_bytes=25165824 total_nvlink_bytes=343915760 weight_to_token_ratio=0.078952 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403272542e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=158752768 dispatch_meta_bytes=228824 combine_payload_bytes=158752768 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=158981592 ingress_bytes=209084416 link_bytes=209084416 token_bytes=317734360 weight_movement_bytes=50331648 total_nvlink_bytes=368066008 weight_to_token_ratio=0.158408 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590890822e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160317440 dispatch_meta_bytes=229432 combine_payload_bytes=160317440 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160546872 ingress_bytes=185483264 link_bytes=185483264 token_bytes=320864312 weight_movement_bytes=25165824 total_nvlink_bytes=346030136 weight_to_token_ratio=0.078431 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411313325e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=160120832 dispatch_meta_bytes=229648 combine_payload_bytes=160120832 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160350480 ingress_bytes=235618304 link_bytes=235618304 token_bytes=320471312 weight_movement_bytes=75497472 total_nvlink_bytes=395968784 weight_to_token_ratio=0.235583 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792783052e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159875072 dispatch_meta_bytes=228912 combine_payload_bytes=159875072 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160103984 ingress_bytes=185040896 link_bytes=185040896 token_bytes=319979056 weight_movement_bytes=25165824 total_nvlink_bytes=345144880 weight_to_token_ratio=0.078648 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407947416e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44528 payload_rows_remote=38920 dup_rows=21008 dup_rows_remote=18272 dedup_ratio=0.320557 router_tokens_per_expert=[504, 497, 510, 487, 497, 491, 532, 530, 545, 541, 521, 515, 510, 501, 535, 516, 501, 528, 507, 523, 521, 510, 496, 482, 532, 495, 507, 472, 487, 504, 547, 522, 478, 478, 497, 510, 494, 493, 494, 512, 524, 562, 535, 503, 516, 521, 523, 510, 477, 538, 531, 525, 543, 526, 525, 507, 498, 509, 487, 468, 541, 516, 513, 548] payload_rows_per_dst=[5578, 5604, 5584, 5500, 5450, 5642, 5608, 5562] dup_rows_per_dst=[2518, 2764, 2552, 2632, 2462, 2746, 2736, 2598] combine_rows_per_src=[5578, 5604, 5584, 5500, 5450, 5642, 5608, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38900 dup_rows=21034 dup_rows_remote=18398 dedup_ratio=0.320953 router_tokens_per_expert=[554, 544, 500, 550, 527, 557, 488, 466, 477, 503, 530, 466, 514, 489, 473, 509, 484, 543, 500, 491, 505, 506, 519, 574, 487, 519, 506, 479, 531, 546, 506, 570, 510, 551, 522, 507, 515, 522, 498, 501, 477, 513, 500, 533, 518, 529, 484, 504, 538, 481, 520, 536, 503, 539, 497, 512, 525, 492, 496, 488, 473, 500, 560, 511] payload_rows_per_dst=[5596, 5412, 5532, 5614, 5602, 5560, 5584, 5602] dup_rows_per_dst=[2776, 2396, 2670, 2788, 2650, 2556, 2562, 2636] combine_rows_per_src=[5596, 5412, 5532, 5614, 5602, 5560, 5584, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44594 payload_rows_remote=39080 dup_rows=20942 dup_rows_remote=18394 dedup_ratio=0.319550 router_tokens_per_expert=[565, 537, 498, 480, 516, 525, 540, 512, 517, 528, 507, 516, 500, 521, 488, 517, 474, 533, 513, 520, 498, 506, 515, 528, 479, 521, 497, 507, 505, 498, 526, 498, 480, 494, 554, 510, 516, 500, 534, 551, 512, 528, 518, 521, 478, 475, 543, 499, 524, 520, 525, 547, 522, 511, 462, 492, 516, 542, 501, 481, 521, 486, 473, 547] payload_rows_per_dst=[5650, 5548, 5532, 5514, 5546, 5624, 5618, 5562] dup_rows_per_dst=[2696, 2640, 2642, 2548, 2732, 2524, 2588, 2572] combine_rows_per_src=[5650, 5548, 5532, 5514, 5546, 5624, 5618, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44384 payload_rows_remote=38874 dup_rows=21152 dup_rows_remote=18618 dedup_ratio=0.322754 router_tokens_per_expert=[507, 515, 495, 484, 473, 495, 511, 499, 513, 500, 506, 510, 495, 520, 541, 541, 509, 507, 527, 505, 548, 508, 500, 498, 540, 493, 509, 554, 506, 483, 531, 490, 532, 502, 480, 485, 491, 517, 494, 521, 504, 500, 513, 584, 511, 506, 520, 530, 534, 505, 547, 524, 531, 514, 485, 486, 502, 475, 510, 541, 557, 532, 522, 500] payload_rows_per_dst=[5442, 5616, 5510, 5532, 5510, 5582, 5584, 5608] dup_rows_per_dst=[2516, 2636, 2694, 2680, 2534, 2754, 2668, 2670] combine_rows_per_src=[5442, 5616, 5510, 5532, 5510, 5582, 5584, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159416320 dispatch_meta_bytes=228768 combine_payload_bytes=159416320 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159645088 ingress_bytes=234913792 link_bytes=234913792 token_bytes=319061408 weight_movement_bytes=75497472 total_nvlink_bytes=394558880 weight_to_token_ratio=0.236624 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787422529e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159334400 dispatch_meta_bytes=229192 combine_payload_bytes=159334400 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159563592 ingress_bytes=184500224 link_bytes=184500224 token_bytes=318897992 weight_movement_bytes=25165824 total_nvlink_bytes=344063816 weight_to_token_ratio=0.078915 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403833526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38946 dup_rows=21068 dup_rows_remote=18452 dedup_ratio=0.321472 router_tokens_per_expert=[511, 530, 529, 485, 511, 532, 528, 477, 519, 497, 548, 494, 495, 542, 504, 489, 515, 524, 511, 488, 516, 514, 484, 525, 486, 534, 516, 462, 536, 502, 511, 483, 502, 519, 527, 516, 495, 525, 491, 536, 532, 483, 482, 532, 533, 490, 510, 507, 489, 519, 544, 553, 512, 527, 501, 496, 525, 514, 500, 482, 564, 504, 518, 542] payload_rows_per_dst=[5636, 5602, 5508, 5522, 5522, 5522, 5610, 5546] dup_rows_per_dst=[2570, 2574, 2646, 2538, 2700, 2616, 2672, 2752] combine_rows_per_src=[5636, 5602, 5508, 5522, 5522, 5522, 5610, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44244 payload_rows_remote=38754 dup_rows=21292 dup_rows_remote=18596 dedup_ratio=0.324890 router_tokens_per_expert=[559, 504, 528, 517, 496, 516, 506, 503, 510, 500, 501, 500, 512, 527, 508, 508, 540, 508, 489, 491, 528, 516, 506, 515, 521, 545, 557, 522, 508, 523, 507, 489, 508, 486, 506, 499, 525, 505, 482, 524, 493, 488, 533, 523, 483, 516, 471, 509, 508, 515, 509, 537, 504, 527, 555, 512, 491, 518, 552, 513, 497, 497, 515, 507] payload_rows_per_dst=[5526, 5482, 5490, 5618, 5524, 5418, 5620, 5566] dup_rows_per_dst=[2732, 2650, 2696, 2726, 2546, 2614, 2714, 2614] combine_rows_per_src=[5526, 5482, 5490, 5618, 5524, 5418, 5620, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38928 dup_rows=21034 dup_rows_remote=18386 dedup_ratio=0.320953 router_tokens_per_expert=[481, 499, 504, 511, 527, 516, 497, 540, 518, 524, 493, 508, 519, 509, 542, 498, 495, 491, 505, 497, 557, 514, 499, 528, 545, 515, 468, 556, 532, 507, 508, 536, 526, 525, 482, 500, 533, 474, 507, 502, 561, 479, 530, 506, 508, 511, 508, 534, 516, 498, 532, 495, 509, 499, 477, 540, 483, 518, 476, 535, 517, 520, 518, 510] payload_rows_per_dst=[5544, 5574, 5602, 5562, 5500, 5602, 5498, 5620] dup_rows_per_dst=[2606, 2648, 2570, 2772, 2598, 2672, 2634, 2534] combine_rows_per_src=[5544, 5574, 5602, 5562, 5500, 5602, 5498, 5620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=160071680 dispatch_meta_bytes=229896 combine_payload_bytes=160071680 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160301576 ingress_bytes=185237504 link_bytes=185237504 token_bytes=320373256 weight_movement_bytes=25165824 total_nvlink_bytes=345539080 weight_to_token_ratio=0.078552 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409443375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44584 payload_rows_remote=38972 dup_rows=20952 dup_rows_remote=18214 dedup_ratio=0.319702 router_tokens_per_expert=[530, 484, 498, 500, 521, 454, 519, 483, 503, 498, 505, 544, 536, 480, 507, 494, 537, 541, 531, 548, 530, 491, 485, 533, 505, 521, 512, 497, 514, 488, 567, 511, 532, 499, 525, 529, 520, 472, 536, 482, 530, 528, 515, 517, 508, 492, 532, 523, 511, 505, 522, 495, 550, 536, 505, 524, 486, 520, 503, 493, 497, 494, 528, 492] payload_rows_per_dst=[5612, 5546, 5670, 5648, 5646, 5450, 5524, 5488] dup_rows_per_dst=[2738, 2614, 2700, 2582, 2848, 2532, 2400, 2538] combine_rows_per_src=[5612, 5546, 5670, 5648, 5646, 5450, 5524, 5488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159227904 dispatch_meta_bytes=229968 combine_payload_bytes=159227904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159457872 ingress_bytes=184393728 link_bytes=184393728 token_bytes=318685776 weight_movement_bytes=25165824 total_nvlink_bytes=343851600 weight_to_token_ratio=0.078968 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403023215e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=229592 combine_payload_bytes=159522816 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159752408 ingress_bytes=209854464 link_bytes=209854464 token_bytes=319275224 weight_movement_bytes=50331648 total_nvlink_bytes=369606872 weight_to_token_ratio=0.157643 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596749998e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159449088 dispatch_meta_bytes=229256 combine_payload_bytes=159449088 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159678344 ingress_bytes=209780736 link_bytes=209780736 token_bytes=319127432 weight_movement_bytes=50331648 total_nvlink_bytes=369459080 weight_to_token_ratio=0.157716 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596189013e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158736384 dispatch_meta_bytes=229400 combine_payload_bytes=158736384 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158965784 ingress_bytes=183902208 link_bytes=183902208 token_bytes=317702168 weight_movement_bytes=25165824 total_nvlink_bytes=342867992 weight_to_token_ratio=0.079212 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399283316e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159629312 dispatch_meta_bytes=228744 combine_payload_bytes=159629312 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159858056 ingress_bytes=184795136 link_bytes=184795136 token_bytes=319487368 weight_movement_bytes=25165824 total_nvlink_bytes=344653192 weight_to_token_ratio=0.078769 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406077466e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=39010 dup_rows=20988 dup_rows_remote=18382 dedup_ratio=0.320251 router_tokens_per_expert=[483, 512, 502, 520, 524, 488, 530, 499, 513, 542, 507, 544, 523, 485, 485, 539, 508, 517, 554, 533, 479, 518, 502, 488, 484, 549, 546, 514, 533, 526, 501, 527, 497, 488, 514, 508, 516, 507, 518, 555, 546, 527, 513, 494, 510, 515, 475, 511, 516, 533, 506, 527, 498, 516, 485, 491, 500, 505, 490, 466, 536, 485, 530, 515] payload_rows_per_dst=[5568, 5596, 5530, 5604, 5542, 5608, 5538, 5562] dup_rows_per_dst=[2548, 2680, 2668, 2756, 2664, 2574, 2606, 2492] combine_rows_per_src=[5568, 5596, 5530, 5604, 5542, 5608, 5538, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44584 payload_rows_remote=38806 dup_rows=20952 dup_rows_remote=18028 dedup_ratio=0.319702 router_tokens_per_expert=[508, 509, 534, 506, 500, 478, 494, 522, 509, 491, 504, 526, 491, 483, 532, 524, 517, 500, 538, 504, 496, 476, 510, 518, 479, 505, 529, 527, 496, 543, 526, 511, 498, 490, 528, 517, 505, 492, 500, 553, 535, 520, 504, 532, 516, 536, 546, 499, 526, 495, 491, 508, 543, 524, 506, 508, 554, 536, 503, 535, 521, 505, 494, 462] payload_rows_per_dst=[5466, 5660, 5548, 5388, 5594, 5650, 5500, 5778] dup_rows_per_dst=[2284, 2812, 2570, 2362, 2560, 2726, 2714, 2924] combine_rows_per_src=[5466, 5660, 5548, 5388, 5594, 5650, 5500, 5778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44302 payload_rows_remote=38720 dup_rows=21234 dup_rows_remote=18580 dedup_ratio=0.324005 router_tokens_per_expert=[505, 532, 501, 526, 524, 474, 547, 483, 495, 492, 469, 509, 505, 518, 487, 516, 540, 523, 505, 495, 528, 487, 523, 497, 518, 511, 498, 522, 531, 518, 525, 495, 486, 494, 495, 487, 526, 508, 565, 472, 512, 485, 558, 492, 499, 544, 497, 501, 490, 510, 534, 509, 496, 542, 521, 577, 510, 503, 540, 495, 548, 537, 518, 518] payload_rows_per_dst=[5520, 5438, 5552, 5582, 5488, 5506, 5580, 5636] dup_rows_per_dst=[2664, 2544, 2644, 2654, 2578, 2670, 2778, 2702] combine_rows_per_src=[5520, 5438, 5552, 5582, 5488, 5506, 5580, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44606 payload_rows_remote=39050 dup_rows=20930 dup_rows_remote=18272 dedup_ratio=0.319366 router_tokens_per_expert=[584, 519, 523, 520, 532, 509, 510, 525, 516, 515, 505, 462, 487, 514, 500, 567, 516, 514, 550, 503, 497, 507, 507, 480, 531, 531, 477, 501, 516, 484, 528, 519, 506, 513, 509, 525, 530, 455, 509, 560, 537, 494, 541, 542, 509, 512, 495, 491, 564, 494, 532, 480, 516, 514, 509, 532, 482, 496, 508, 491, 509, 492, 507, 465] payload_rows_per_dst=[5640, 5566, 5602, 5588, 5556, 5580, 5594, 5480] dup_rows_per_dst=[2804, 2566, 2546, 2586, 2658, 2662, 2688, 2420] combine_rows_per_src=[5640, 5566, 5602, 5588, 5556, 5580, 5594, 5480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159784960 dispatch_meta_bytes=229568 combine_payload_bytes=159784960 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160014528 ingress_bytes=210116608 link_bytes=210116608 token_bytes=319799488 weight_movement_bytes=50331648 total_nvlink_bytes=370131136 weight_to_token_ratio=0.157385 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598744611e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158949376 dispatch_meta_bytes=227336 combine_payload_bytes=158949376 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159176712 ingress_bytes=184115200 link_bytes=184115200 token_bytes=318126088 weight_movement_bytes=25165824 total_nvlink_bytes=343291912 weight_to_token_ratio=0.079106 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400903939e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44550 payload_rows_remote=39012 dup_rows=20986 dup_rows_remote=18398 dedup_ratio=0.320221 router_tokens_per_expert=[508, 508, 525, 504, 497, 486, 513, 509, 513, 563, 485, 529, 494, 538, 485, 522, 488, 510, 489, 507, 523, 539, 502, 528, 515, 543, 493, 534, 498, 570, 483, 501, 511, 546, 545, 485, 505, 511, 525, 521, 500, 480, 533, 519, 492, 535, 509, 495, 535, 497, 475, 546, 520, 483, 512, 533, 497, 497, 506, 520, 522, 512, 502, 497] payload_rows_per_dst=[5524, 5606, 5520, 5596, 5616, 5538, 5606, 5544] dup_rows_per_dst=[2576, 2652, 2652, 2678, 2682, 2588, 2596, 2562] combine_rows_per_src=[5524, 5606, 5520, 5596, 5616, 5538, 5606, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38918 dup_rows=21098 dup_rows_remote=18464 dedup_ratio=0.321930 router_tokens_per_expert=[500, 562, 502, 530, 504, 477, 526, 487, 504, 446, 530, 518, 523, 520, 504, 532, 542, 497, 531, 481, 462, 531, 501, 521, 530, 531, 499, 534, 538, 510, 515, 456, 527, 499, 515, 522, 506, 505, 525, 536, 534, 499, 506, 524, 530, 517, 498, 508, 513, 477, 499, 489, 496, 503, 543, 517, 511, 509, 483, 510, 526, 526, 549, 522] payload_rows_per_dst=[5580, 5520, 5594, 5566, 5582, 5546, 5506, 5544] dup_rows_per_dst=[2596, 2634, 2538, 2660, 2688, 2686, 2568, 2728] combine_rows_per_src=[5580, 5520, 5594, 5566, 5582, 5546, 5506, 5544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158597120 dispatch_meta_bytes=229200 combine_payload_bytes=158597120 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158826320 ingress_bytes=208928768 link_bytes=208928768 token_bytes=317423440 weight_movement_bytes=50331648 total_nvlink_bytes=367755088 weight_to_token_ratio=0.158563 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589706521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44634 payload_rows_remote=39022 dup_rows=20902 dup_rows_remote=18312 dedup_ratio=0.318939 router_tokens_per_expert=[464, 501, 506, 497, 480, 506, 499, 501, 556, 487, 549, 551, 496, 498, 489, 489, 543, 488, 510, 505, 508, 555, 510, 482, 490, 542, 532, 499, 507, 491, 528, 514, 481, 488, 529, 517, 517, 542, 547, 513, 532, 503, 487, 478, 506, 516, 498, 524, 539, 527, 549, 508, 523, 529, 495, 508, 513, 512, 503, 533, 516, 495, 542, 525] payload_rows_per_dst=[5550, 5580, 5612, 5600, 5590, 5496, 5628, 5578] dup_rows_per_dst=[2358, 2650, 2590, 2606, 2678, 2592, 2728, 2700] combine_rows_per_src=[5550, 5580, 5612, 5600, 5590, 5496, 5628, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38822 dup_rows=21050 dup_rows_remote=18206 dedup_ratio=0.321198 router_tokens_per_expert=[544, 531, 510, 502, 560, 508, 509, 537, 519, 475, 504, 545, 512, 491, 504, 515, 495, 500, 540, 531, 535, 484, 483, 509, 478, 518, 517, 503, 532, 486, 521, 523, 542, 527, 552, 524, 520, 522, 533, 470, 504, 511, 515, 539, 530, 499, 530, 534, 492, 496, 499, 503, 491, 508, 513, 501, 496, 509, 509, 502, 501, 483, 509, 483] payload_rows_per_dst=[5664, 5494, 5692, 5596, 5612, 5534, 5468, 5426] dup_rows_per_dst=[2844, 2636, 2694, 2576, 2646, 2636, 2460, 2558] combine_rows_per_src=[5664, 5494, 5692, 5596, 5612, 5534, 5468, 5426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159948800 dispatch_meta_bytes=229288 combine_payload_bytes=159948800 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160178088 ingress_bytes=235446272 link_bytes=235446272 token_bytes=320126888 weight_movement_bytes=75497472 total_nvlink_bytes=395624360 weight_to_token_ratio=0.235836 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.791474087e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159793152 dispatch_meta_bytes=229640 combine_payload_bytes=159793152 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160022792 ingress_bytes=210124800 link_bytes=210124800 token_bytes=319815944 weight_movement_bytes=50331648 total_nvlink_bytes=370147592 weight_to_token_ratio=0.157377 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598806942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=229528 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159637656 ingress_bytes=184573952 link_bytes=184573952 token_bytes=319045784 weight_movement_bytes=25165824 total_nvlink_bytes=344211608 weight_to_token_ratio=0.078878 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404394511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159834112 dispatch_meta_bytes=229336 combine_payload_bytes=159834112 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160063448 ingress_bytes=184999936 link_bytes=184999936 token_bytes=319897560 weight_movement_bytes=25165824 total_nvlink_bytes=345063384 weight_to_token_ratio=0.078668 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407635757e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159014912 dispatch_meta_bytes=228112 combine_payload_bytes=159014912 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159243024 ingress_bytes=209346560 link_bytes=209346560 token_bytes=318257936 weight_movement_bytes=50331648 total_nvlink_bytes=368589584 weight_to_token_ratio=0.158147 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592885435e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44558 payload_rows_remote=39002 dup_rows=20978 dup_rows_remote=18404 dedup_ratio=0.320099 router_tokens_per_expert=[514, 465, 519, 489, 519, 505, 508, 488, 523, 505, 502, 487, 516, 535, 505, 497, 528, 487, 495, 496, 515, 495, 512, 530, 526, 516, 527, 514, 516, 540, 535, 523, 541, 545, 520, 495, 517, 528, 538, 521, 522, 512, 527, 539, 511, 488, 500, 489, 477, 505, 505, 529, 515, 511, 503, 520, 506, 501, 558, 476, 487, 495, 530, 525] payload_rows_per_dst=[5498, 5552, 5488, 5642, 5662, 5552, 5556, 5608] dup_rows_per_dst=[2516, 2588, 2628, 2752, 2748, 2624, 2574, 2548] combine_rows_per_src=[5498, 5552, 5488, 5642, 5662, 5552, 5556, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38880 dup_rows=21068 dup_rows_remote=18352 dedup_ratio=0.321472 router_tokens_per_expert=[487, 477, 538, 525, 513, 471, 522, 532, 535, 552, 501, 489, 515, 481, 555, 527, 503, 514, 509, 535, 515, 497, 512, 515, 499, 549, 503, 452, 513, 495, 530, 494, 513, 563, 534, 503, 536, 532, 495, 528, 502, 512, 523, 480, 526, 506, 491, 506, 499, 507, 530, 499, 458, 535, 543, 496, 525, 485, 513, 546, 543, 509, 471, 504] payload_rows_per_dst=[5562, 5480, 5582, 5532, 5576, 5540, 5608, 5588] dup_rows_per_dst=[2568, 2598, 2618, 2524, 2734, 2680, 2630, 2716] combine_rows_per_src=[5562, 5480, 5582, 5532, 5576, 5540, 5608, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44682 payload_rows_remote=39066 dup_rows=20854 dup_rows_remote=18282 dedup_ratio=0.318207 router_tokens_per_expert=[480, 511, 514, 513, 488, 503, 523, 524, 489, 511, 529, 527, 525, 518, 538, 525, 524, 510, 490, 481, 571, 494, 555, 546, 491, 538, 516, 521, 527, 528, 487, 486, 516, 526, 499, 549, 525, 449, 515, 505, 493, 492, 518, 496, 504, 525, 505, 450, 532, 528, 522, 517, 516, 486, 550, 531, 479, 497, 513, 527, 506, 499, 533, 482] payload_rows_per_dst=[5510, 5642, 5552, 5616, 5698, 5474, 5662, 5528] dup_rows_per_dst=[2602, 2682, 2790, 2572, 2470, 2492, 2702, 2544] combine_rows_per_src=[5510, 5642, 5552, 5616, 5698, 5474, 5662, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5105416616 prefetch_bytes=402653184 reduce_bytes=327155712 weight_movement_bytes=729808896 egress_bytes=2554542504 ingress_bytes=3280683008 link_bytes=3280683008 total_nvlink_bytes=5835225512 weight_to_token_ratio=0.142948 prefetch_share_of_ingress=0.122735 dedup_ratio=0.321030 prefetch_slots=32 unused_prefetch_slots=224 slot_utilization=0.125000 reduce_slots=13 remote_demand=16 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=20320 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.560137942e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44672 payload_rows_remote=39076 dup_rows=20864 dup_rows_remote=18078 dedup_ratio=0.318359 router_tokens_per_expert=[522, 467, 518, 522, 549, 515, 522, 542, 494, 515, 510, 501, 520, 494, 511, 528, 532, 502, 488, 491, 485, 503, 493, 513, 490, 499, 538, 498, 550, 527, 479, 509, 555, 568, 518, 483, 494, 504, 517, 552, 501, 537, 514, 527, 496, 466, 531, 509, 518, 497, 534, 484, 511, 509, 507, 498, 510, 507, 516, 497, 513, 528, 537, 503] payload_rows_per_dst=[5678, 5592, 5570, 5602, 5596, 5512, 5532, 5590] dup_rows_per_dst=[2636, 2554, 2444, 2578, 2786, 2650, 2584, 2632] combine_rows_per_src=[5678, 5592, 5570, 5602, 5596, 5512, 5532, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159252480 dispatch_meta_bytes=228928 combine_payload_bytes=159252480 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159481408 ingress_bytes=184418304 link_bytes=184418304 token_bytes=318733888 weight_movement_bytes=25165824 total_nvlink_bytes=343899712 weight_to_token_ratio=0.078956 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403210210e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159752192 dispatch_meta_bytes=229624 combine_payload_bytes=159752192 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159981816 ingress_bytes=210083840 link_bytes=210083840 token_bytes=319734008 weight_movement_bytes=50331648 total_nvlink_bytes=370065656 weight_to_token_ratio=0.157417 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598495284e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44588 payload_rows_remote=38910 dup_rows=20948 dup_rows_remote=18264 dedup_ratio=0.319641 router_tokens_per_expert=[511, 518, 516, 542, 525, 528, 512, 511, 466, 537, 550, 506, 534, 519, 507, 472, 496, 487, 503, 512, 466, 505, 515, 496, 512, 522, 548, 510, 522, 471, 478, 523, 507, 561, 484, 512, 518, 522, 507, 495, 519, 500, 540, 559, 493, 490, 534, 546, 524, 503, 496, 533, 513, 495, 538, 504, 494, 505, 507, 492, 484, 497, 544, 532] payload_rows_per_dst=[5618, 5526, 5440, 5616, 5570, 5678, 5576, 5564] dup_rows_per_dst=[2708, 2656, 2520, 2556, 2642, 2684, 2636, 2546] combine_rows_per_src=[5618, 5526, 5440, 5616, 5570, 5678, 5576, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44314 payload_rows_remote=38776 dup_rows=21222 dup_rows_remote=18574 dedup_ratio=0.323822 router_tokens_per_expert=[518, 511, 512, 509, 488, 484, 504, 521, 504, 518, 515, 489, 520, 516, 504, 527, 477, 548, 533, 520, 502, 516, 478, 538, 473, 479, 523, 527, 499, 501, 515, 526, 471, 525, 491, 527, 486, 557, 497, 508, 538, 551, 520, 508, 487, 539, 518, 508, 527, 496, 470, 498, 516, 500, 543, 519, 547, 502, 513, 537, 514, 512, 537, 511] payload_rows_per_dst=[5464, 5538, 5522, 5508, 5530, 5592, 5534, 5626] dup_rows_per_dst=[2630, 2648, 2702, 2578, 2594, 2746, 2604, 2720] combine_rows_per_src=[5464, 5538, 5522, 5508, 5530, 5592, 5534, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44428 payload_rows_remote=38688 dup_rows=21108 dup_rows_remote=18210 dedup_ratio=0.322083 router_tokens_per_expert=[503, 497, 493, 509, 509, 482, 534, 522, 516, 513, 527, 500, 525, 493, 512, 517, 502, 484, 493, 500, 523, 515, 505, 532, 525, 527, 524, 501, 532, 523, 502, 515, 493, 513, 525, 522, 505, 509, 495, 539, 483, 527, 497, 529, 498, 512, 544, 522, 540, 515, 526, 533, 519, 503, 514, 526, 467, 503, 512, 490, 496, 487, 540, 529] payload_rows_per_dst=[5740, 5528, 5666, 5628, 5222, 5586, 5482, 5576] dup_rows_per_dst=[2898, 2722, 2812, 2626, 2440, 2638, 2500, 2472] combine_rows_per_src=[5740, 5528, 5666, 5628, 5222, 5586, 5482, 5576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=229392 combine_payload_bytes=160014336 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160243728 ingress_bytes=210345984 link_bytes=210345984 token_bytes=320258064 weight_movement_bytes=50331648 total_nvlink_bytes=370589712 weight_to_token_ratio=0.157160 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600489897e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5100048648 prefetch_bytes=377487360 reduce_bytes=201326592 weight_movement_bytes=578813952 egress_bytes=2551853320 ingress_bytes=3127009280 link_bytes=3127009280 total_nvlink_bytes=5678862600 weight_to_token_ratio=0.113492 prefetch_share_of_ingress=0.120718 dedup_ratio=0.320719 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=8 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041131 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.487057973e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5106990560 prefetch_bytes=327155712 reduce_bytes=478150656 weight_movement_bytes=805306368 egress_bytes=2555330016 ingress_bytes=3356966912 link_bytes=3356966912 total_nvlink_bytes=5912296928 weight_to_token_ratio=0.157687 prefetch_share_of_ingress=0.097456 dedup_ratio=0.321003 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=19 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.596414965e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=160055296 dispatch_meta_bytes=228616 combine_payload_bytes=160055296 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160283912 ingress_bytes=235552768 link_bytes=235552768 token_bytes=320339208 weight_movement_bytes=75497472 total_nvlink_bytes=395836680 weight_to_token_ratio=0.235680 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.792284398e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159375360 dispatch_meta_bytes=228696 combine_payload_bytes=159375360 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159604056 ingress_bytes=184541184 link_bytes=184541184 token_bytes=318979416 weight_movement_bytes=25165824 total_nvlink_bytes=344145240 weight_to_token_ratio=0.078895 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404145185e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=229400 combine_payload_bytes=158826496 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159055896 ingress_bytes=234323968 link_bytes=234323968 token_bytes=317882392 weight_movement_bytes=75497472 total_nvlink_bytes=393379864 weight_to_token_ratio=0.237501 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782934650e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5106221656 prefetch_bytes=301989888 reduce_bytes=452984832 weight_movement_bytes=754974720 egress_bytes=2554946136 ingress_bytes=3306250240 link_bytes=3306250240 total_nvlink_bytes=5861196376 weight_to_token_ratio=0.147854 prefetch_share_of_ingress=0.091339 dedup_ratio=0.320995 prefetch_slots=24 unused_prefetch_slots=232 slot_utilization=0.093750 reduce_slots=18 remote_demand=12 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=19812 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.572296510e-04 prefetch_bytes_per_gemm_flop=1.436121324e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158466048 dispatch_meta_bytes=227592 combine_payload_bytes=158466048 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158693640 ingress_bytes=183631872 link_bytes=183631872 token_bytes=317159688 weight_movement_bytes=25165824 total_nvlink_bytes=342325512 weight_to_token_ratio=0.079347 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397226371e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5114050776 prefetch_bytes=377487360 reduce_bytes=276824064 weight_movement_bytes=654311424 egress_bytes=2558859480 ingress_bytes=3209502720 link_bytes=3209502720 total_nvlink_bytes=5768362200 weight_to_token_ratio=0.127944 prefetch_share_of_ingress=0.117616 dedup_ratio=0.319859 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=11 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040671 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.526287958e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5101893824 prefetch_bytes=327155712 reduce_bytes=377487360 weight_movement_bytes=704643072 egress_bytes=2552780992 ingress_bytes=3253755904 link_bytes=3253755904 total_nvlink_bytes=5806536896 weight_to_token_ratio=0.138114 prefetch_share_of_ingress=0.100547 dedup_ratio=0.321262 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=15 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.547332683e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5109267464 prefetch_bytes=377487360 reduce_bytes=327155712 weight_movement_bytes=704643072 egress_bytes=2556468232 ingress_bytes=3257442304 link_bytes=3257442304 total_nvlink_bytes=5813910536 weight_to_token_ratio=0.137915 prefetch_share_of_ingress=0.115885 dedup_ratio=0.320766 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=13 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.549085760e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5096203472 prefetch_bytes=327155712 reduce_bytes=377487360 weight_movement_bytes=704643072 egress_bytes=2549933264 ingress_bytes=3250913280 link_bytes=3250913280 total_nvlink_bytes=5800846544 weight_to_token_ratio=0.138268 prefetch_share_of_ingress=0.100635 dedup_ratio=0.321659 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=15 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.545980865e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none + [2026-09-21 08:32:50] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 211.7 | throughput per GPU (TFLOP/s/GPU): 152.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.688962E+00 | loss scale: 1.0 | grad norm: 3.332 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44496 payload_rows_remote=38944 dup_rows=21040 dup_rows_remote=18414 dedup_ratio=0.321045 router_tokens_per_expert=[520, 526, 513, 531, 480, 492, 513, 501, 462, 505, 521, 525, 525, 527, 548, 481, 456, 522, 490, 508, 549, 528, 502, 486, 515, 517, 523, 505, 467, 534, 528, 490, 521, 521, 482, 527, 515, 550, 555, 524, 527, 550, 505, 514, 508, 493, 490, 480, 451, 522, 541, 527, 525, 510, 508, 505, 483, 493, 516, 513, 571, 518, 528, 505] payload_rows_per_dst=[5610, 5512, 5458, 5516, 5700, 5566, 5552, 5582] dup_rows_per_dst=[2542, 2676, 2624, 2642, 2690, 2568, 2626, 2672] combine_rows_per_src=[5610, 5512, 5458, 5516, 5700, 5566, 5552, 5582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44608 payload_rows_remote=39066 dup_rows=20928 dup_rows_remote=18270 dedup_ratio=0.319336 router_tokens_per_expert=[503, 482, 499, 514, 520, 487, 484, 479, 497, 513, 548, 496, 541, 488, 506, 549, 551, 542, 505, 504, 484, 491, 522, 520, 527, 526, 526, 506, 489, 495, 564, 525, 503, 521, 486, 508, 547, 518, 485, 531, 504, 527, 478, 475, 489, 561, 546, 520, 507, 519, 511, 502, 523, 539, 526, 484, 498, 536, 475, 496, 512, 537, 504, 517] payload_rows_per_dst=[5480, 5626, 5654, 5544, 5632, 5542, 5580, 5550] dup_rows_per_dst=[2456, 2650, 2584, 2772, 2566, 2658, 2642, 2600] combine_rows_per_src=[5480, 5626, 5654, 5544, 5632, 5542, 5580, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44514 payload_rows_remote=38936 dup_rows=21022 dup_rows_remote=18426 dedup_ratio=0.320770 router_tokens_per_expert=[493, 527, 569, 536, 522, 529, 524, 521, 516, 540, 506, 491, 476, 513, 524, 485, 536, 524, 507, 495, 536, 523, 538, 502, 509, 475, 525, 482, 494, 527, 537, 538, 523, 480, 531, 538, 463, 516, 467, 497, 493, 520, 524, 492, 535, 516, 519, 495, 496, 507, 478, 554, 471, 517, 529, 509, 534, 516, 489, 494, 540, 511, 495, 499] payload_rows_per_dst=[5712, 5464, 5646, 5578, 5472, 5582, 5562, 5498] dup_rows_per_dst=[2730, 2638, 2676, 2596, 2558, 2606, 2560, 2658] combine_rows_per_src=[5712, 5464, 5646, 5578, 5472, 5582, 5562, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44712 payload_rows_remote=39118 dup_rows=20824 dup_rows_remote=18228 dedup_ratio=0.317749 router_tokens_per_expert=[498, 530, 496, 489, 515, 524, 488, 515, 481, 480, 512, 523, 486, 527, 547, 530, 516, 524, 490, 507, 508, 499, 481, 510, 481, 505, 513, 536, 500, 512, 519, 509, 505, 532, 480, 553, 518, 526, 510, 471, 506, 479, 514, 551, 555, 532, 507, 512, 534, 502, 547, 521, 528, 508, 500, 481, 508, 512, 509, 517, 561, 502, 492, 544] payload_rows_per_dst=[5564, 5522, 5520, 5552, 5594, 5656, 5606, 5698] dup_rows_per_dst=[2546, 2650, 2550, 2598, 2596, 2656, 2636, 2592] combine_rows_per_src=[5564, 5522, 5520, 5552, 5594, 5656, 5606, 5698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44594 payload_rows_remote=38942 dup_rows=20942 dup_rows_remote=18166 dedup_ratio=0.319550 router_tokens_per_expert=[559, 499, 508, 527, 491, 510, 516, 477, 474, 469, 502, 505, 502, 518, 504, 511, 511, 502, 520, 546, 532, 550, 532, 521, 494, 520, 504, 487, 505, 538, 494, 535, 514, 481, 505, 485, 493, 504, 516, 519, 505, 515, 507, 501, 515, 507, 505, 535, 498, 529, 520, 541, 520, 502, 518, 525, 506, 496, 518, 503, 521, 583, 508, 510] payload_rows_per_dst=[5568, 5526, 5652, 5536, 5512, 5532, 5634, 5634] dup_rows_per_dst=[2606, 2444, 2776, 2618, 2522, 2648, 2672, 2656] combine_rows_per_src=[5568, 5526, 5652, 5536, 5512, 5532, 5634, 5634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=38816 dup_rows=21070 dup_rows_remote=18302 dedup_ratio=0.321503 router_tokens_per_expert=[495, 470, 507, 536, 539, 496, 500, 546, 503, 526, 457, 502, 499, 524, 522, 457, 508, 520, 543, 535, 504, 493, 512, 566, 528, 507, 522, 513, 484, 472, 506, 504, 456, 498, 503, 482, 525, 523, 554, 521, 497, 529, 507, 523, 484, 496, 541, 500, 493, 511, 529, 536, 490, 519, 540, 506, 534, 514, 534, 530, 547, 484, 541, 525] payload_rows_per_dst=[5478, 5478, 5660, 5478, 5596, 5532, 5594, 5650] dup_rows_per_dst=[2700, 2502, 2702, 2594, 2528, 2622, 2654, 2768] combine_rows_per_src=[5478, 5478, 5660, 5478, 5596, 5532, 5594, 5650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44690 payload_rows_remote=39152 dup_rows=20846 dup_rows_remote=18298 dedup_ratio=0.318085 router_tokens_per_expert=[531, 551, 533, 522, 501, 470, 502, 486, 513, 506, 482, 510, 519, 527, 484, 502, 549, 506, 511, 515, 500, 501, 516, 510, 534, 459, 518, 506, 536, 530, 500, 525, 489, 509, 546, 496, 548, 518, 562, 516, 523, 501, 522, 522, 495, 488, 499, 493, 531, 491, 512, 496, 535, 526, 473, 515, 517, 506, 516, 524, 514, 516, 526, 488] payload_rows_per_dst=[5612, 5538, 5594, 5616, 5648, 5516, 5570, 5596] dup_rows_per_dst=[2580, 2548, 2622, 2600, 2720, 2570, 2588, 2618] combine_rows_per_src=[5612, 5538, 5594, 5616, 5648, 5516, 5570, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159514624 dispatch_meta_bytes=229432 combine_payload_bytes=159514624 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159744056 ingress_bytes=209846272 link_bytes=209846272 token_bytes=319258680 weight_movement_bytes=50331648 total_nvlink_bytes=369590328 weight_to_token_ratio=0.157652 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596687666e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160014336 dispatch_meta_bytes=229344 combine_payload_bytes=160014336 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160243680 ingress_bytes=185180160 link_bytes=185180160 token_bytes=320258016 weight_movement_bytes=25165824 total_nvlink_bytes=345423840 weight_to_token_ratio=0.078580 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409007054e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159481856 dispatch_meta_bytes=229448 combine_payload_bytes=159481856 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159711304 ingress_bytes=184647680 link_bytes=184647680 token_bytes=319193160 weight_movement_bytes=25165824 total_nvlink_bytes=344358984 weight_to_token_ratio=0.078842 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404955496e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160227328 dispatch_meta_bytes=229384 combine_payload_bytes=160227328 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160456712 ingress_bytes=210558976 link_bytes=210558976 token_bytes=320684040 weight_movement_bytes=50331648 total_nvlink_bytes=371015688 weight_to_token_ratio=0.156951 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602110520e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=228432 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159734864 ingress_bytes=209838080 link_bytes=209838080 token_bytes=319241296 weight_movement_bytes=50331648 total_nvlink_bytes=369572944 weight_to_token_ratio=0.157660 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596625334e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=158990336 dispatch_meta_bytes=228472 combine_payload_bytes=158990336 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159218808 ingress_bytes=259653632 link_bytes=259653632 token_bytes=318209144 weight_movement_bytes=100663296 total_nvlink_bytes=418872440 weight_to_token_ratio=0.316343 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975664126e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=160366592 dispatch_meta_bytes=229800 combine_payload_bytes=160366592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160596392 ingress_bytes=185532416 link_bytes=185532416 token_bytes=320962984 weight_movement_bytes=25165824 total_nvlink_bytes=346128808 weight_to_token_ratio=0.078407 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411687315e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44580 payload_rows_remote=38954 dup_rows=20956 dup_rows_remote=18230 dedup_ratio=0.319763 router_tokens_per_expert=[537, 520, 530, 491, 505, 503, 484, 533, 498, 519, 458, 527, 528, 520, 508, 534, 519, 526, 534, 508, 489, 503, 506, 497, 540, 503, 497, 496, 517, 458, 539, 565, 551, 514, 526, 520, 516, 526, 509, 507, 493, 535, 501, 532, 524, 499, 492, 493, 550, 500, 502, 543, 504, 494, 500, 497, 517, 514, 483, 504, 526, 511, 495, 498] payload_rows_per_dst=[5626, 5818, 5424, 5652, 5532, 5608, 5574, 5346] dup_rows_per_dst=[2726, 2944, 2394, 2644, 2670, 2674, 2486, 2418] combine_rows_per_src=[5626, 5818, 5424, 5652, 5532, 5608, 5574, 5346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44516 payload_rows_remote=38988 dup_rows=21020 dup_rows_remote=18454 dedup_ratio=0.320740 router_tokens_per_expert=[570, 520, 531, 539, 511, 492, 520, 481, 522, 491, 487, 531, 509, 503, 510, 471, 512, 572, 501, 513, 506, 528, 530, 518, 492, 496, 537, 508, 501, 528, 506, 491, 503, 519, 495, 483, 480, 500, 505, 500, 486, 498, 508, 541, 465, 511, 494, 544, 516, 492, 526, 493, 499, 556, 561, 531, 526, 545, 548, 499, 512, 503, 525, 477] payload_rows_per_dst=[5610, 5464, 5710, 5500, 5494, 5528, 5602, 5608] dup_rows_per_dst=[2718, 2584, 2650, 2618, 2476, 2566, 2746, 2662] combine_rows_per_src=[5610, 5464, 5710, 5500, 5494, 5528, 5602, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44444 payload_rows_remote=38838 dup_rows=21092 dup_rows_remote=18466 dedup_ratio=0.321838 router_tokens_per_expert=[516, 495, 556, 518, 490, 490, 530, 513, 496, 526, 531, 502, 490, 477, 484, 504, 509, 511, 504, 495, 516, 518, 544, 538, 502, 516, 561, 488, 502, 497, 516, 534, 519, 487, 552, 553, 488, 514, 490, 499, 454, 510, 538, 521, 540, 528, 495, 514, 501, 504, 516, 535, 533, 497, 510, 494, 491, 566, 492, 526, 497, 497, 534, 504] payload_rows_per_dst=[5584, 5456, 5582, 5606, 5564, 5550, 5532, 5570] dup_rows_per_dst=[2632, 2564, 2688, 2626, 2640, 2650, 2648, 2644] combine_rows_per_src=[5584, 5456, 5582, 5606, 5564, 5550, 5532, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44386 payload_rows_remote=38882 dup_rows=21150 dup_rows_remote=18712 dedup_ratio=0.322723 router_tokens_per_expert=[519, 517, 490, 512, 522, 526, 518, 491, 491, 523, 491, 562, 516, 520, 495, 530, 507, 523, 515, 507, 496, 512, 488, 482, 518, 538, 532, 481, 515, 488, 560, 539, 502, 509, 503, 518, 488, 494, 521, 534, 553, 527, 521, 492, 515, 531, 522, 517, 511, 508, 465, 468, 507, 502, 510, 500, 521, 528, 554, 485, 498, 490, 545, 505] payload_rows_per_dst=[5510, 5572, 5478, 5626, 5510, 5648, 5504, 5538] dup_rows_per_dst=[2680, 2684, 2582, 2716, 2628, 2708, 2438, 2714] combine_rows_per_src=[5510, 5572, 5478, 5626, 5510, 5648, 5504, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44578 payload_rows_remote=38922 dup_rows=20958 dup_rows_remote=18336 dedup_ratio=0.319794 router_tokens_per_expert=[494, 484, 502, 527, 537, 490, 537, 529, 507, 543, 489, 502, 494, 548, 501, 538, 519, 511, 522, 521, 537, 488, 504, 485, 530, 556, 537, 472, 483, 521, 511, 475, 488, 525, 560, 534, 499, 527, 496, 510, 488, 509, 489, 538, 543, 473, 497, 551, 520, 522, 526, 515, 492, 480, 520, 502, 514, 510, 496, 519, 530, 521, 487, 493] payload_rows_per_dst=[5542, 5542, 5640, 5528, 5656, 5526, 5616, 5528] dup_rows_per_dst=[2658, 2702, 2534, 2642, 2622, 2650, 2538, 2612] combine_rows_per_src=[5542, 5542, 5640, 5528, 5656, 5526, 5616, 5528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44482 payload_rows_remote=38976 dup_rows=21054 dup_rows_remote=18430 dedup_ratio=0.321259 router_tokens_per_expert=[522, 489, 502, 517, 541, 503, 519, 508, 504, 506, 507, 510, 546, 516, 511, 529, 514, 496, 529, 482, 498, 524, 488, 534, 557, 484, 532, 525, 502, 483, 483, 543, 504, 531, 481, 501, 536, 522, 504, 531, 552, 510, 522, 517, 505, 521, 492, 518, 484, 500, 512, 556, 473, 508, 513, 483, 470, 542, 505, 520, 510, 500, 532, 509] payload_rows_per_dst=[5588, 5596, 5506, 5586, 5616, 5562, 5504, 5524] dup_rows_per_dst=[2614, 2662, 2624, 2632, 2604, 2712, 2554, 2652] combine_rows_per_src=[5588, 5596, 5506, 5586, 5616, 5562, 5504, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44364 payload_rows_remote=38740 dup_rows=21172 dup_rows_remote=18468 dedup_ratio=0.323059 router_tokens_per_expert=[471, 505, 474, 513, 527, 539, 488, 523, 480, 531, 501, 519, 496, 486, 497, 517, 521, 586, 541, 513, 494, 514, 478, 519, 498, 516, 483, 512, 513, 509, 525, 532, 495, 521, 497, 481, 511, 496, 524, 537, 529, 543, 492, 495, 498, 547, 504, 552, 542, 497, 448, 522, 518, 532, 503, 504, 529, 538, 491, 494, 544, 559, 506, 498] payload_rows_per_dst=[5280, 5480, 5608, 5580, 5528, 5578, 5686, 5624] dup_rows_per_dst=[2432, 2574, 2704, 2596, 2596, 2752, 2814, 2704] combine_rows_per_src=[5280, 5480, 5608, 5580, 5528, 5578, 5686, 5624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38790 dup_rows=21142 dup_rows_remote=18422 dedup_ratio=0.322601 router_tokens_per_expert=[527, 507, 517, 530, 534, 534, 520, 507, 541, 519, 501, 496, 481, 545, 540, 539, 531, 545, 476, 531, 489, 512, 494, 512, 529, 512, 534, 511, 542, 524, 506, 487, 558, 522, 498, 482, 491, 526, 502, 526, 524, 489, 467, 486, 527, 525, 503, 482, 466, 510, 546, 494, 447, 563, 521, 510, 485, 463, 527, 501, 530, 524, 512, 488] payload_rows_per_dst=[5636, 5604, 5572, 5548, 5556, 5492, 5484, 5502] dup_rows_per_dst=[2716, 2720, 2608, 2742, 2654, 2514, 2630, 2558] combine_rows_per_src=[5636, 5604, 5572, 5548, 5556, 5492, 5484, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=228736 combine_payload_bytes=159555584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159784320 ingress_bytes=184721408 link_bytes=184721408 token_bytes=319339904 weight_movement_bytes=25165824 total_nvlink_bytes=344505728 weight_to_token_ratio=0.078806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405516481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=229768 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159924616 ingress_bytes=210026496 link_bytes=210026496 token_bytes=319619464 weight_movement_bytes=50331648 total_nvlink_bytes=369951112 weight_to_token_ratio=0.157474 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598058962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159080448 dispatch_meta_bytes=229216 combine_payload_bytes=159080448 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159309664 ingress_bytes=209412096 link_bytes=209412096 token_bytes=318390112 weight_movement_bytes=50331648 total_nvlink_bytes=368721760 weight_to_token_ratio=0.158082 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593384088e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159260672 dispatch_meta_bytes=230376 combine_payload_bytes=159260672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159491048 ingress_bytes=184426496 link_bytes=184426496 token_bytes=318751720 weight_movement_bytes=25165824 total_nvlink_bytes=343917544 weight_to_token_ratio=0.078951 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403272542e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158679040 dispatch_meta_bytes=228832 combine_payload_bytes=158679040 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158907872 ingress_bytes=209010688 link_bytes=209010688 token_bytes=317586912 weight_movement_bytes=50331648 total_nvlink_bytes=367918560 weight_to_token_ratio=0.158481 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590329837e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=229032 combine_payload_bytes=159424512 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159653544 ingress_bytes=184590336 link_bytes=184590336 token_bytes=319078056 weight_movement_bytes=25165824 total_nvlink_bytes=344243880 weight_to_token_ratio=0.078870 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404519175e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=229624 combine_payload_bytes=159645696 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159875320 ingress_bytes=235143168 link_bytes=235143168 token_bytes=319521016 weight_movement_bytes=75497472 total_nvlink_bytes=395018488 weight_to_token_ratio=0.236283 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.789167816e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158883840 dispatch_meta_bytes=228848 combine_payload_bytes=158883840 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159112688 ingress_bytes=184049664 link_bytes=184049664 token_bytes=317996528 weight_movement_bytes=25165824 total_nvlink_bytes=343162352 weight_to_token_ratio=0.079139 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400405285e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44468 payload_rows_remote=38944 dup_rows=21068 dup_rows_remote=18476 dedup_ratio=0.321472 router_tokens_per_expert=[514, 510, 546, 498, 514, 506, 496, 547, 553, 533, 525, 496, 495, 521, 527, 490, 560, 496, 478, 519, 523, 503, 468, 511, 519, 536, 495, 510, 524, 510, 502, 500, 505, 562, 526, 497, 496, 534, 498, 526, 552, 486, 537, 497, 476, 515, 518, 477, 482, 470, 523, 528, 544, 520, 529, 516, 531, 504, 484, 485, 496, 503, 521, 505] payload_rows_per_dst=[5560, 5592, 5582, 5566, 5624, 5524, 5598, 5422] dup_rows_per_dst=[2702, 2688, 2534, 2626, 2664, 2592, 2626, 2636] combine_rows_per_src=[5560, 5592, 5582, 5566, 5624, 5524, 5598, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=38856 dup_rows=21026 dup_rows_remote=18364 dedup_ratio=0.320831 router_tokens_per_expert=[492, 530, 505, 537, 514, 486, 494, 480, 518, 503, 512, 504, 513, 530, 497, 535, 526, 508, 458, 511, 534, 475, 513, 513, 500, 506, 542, 533, 529, 530, 491, 527, 534, 533, 525, 505, 480, 514, 517, 515, 477, 530, 537, 504, 528, 505, 526, 501, 499, 519, 536, 513, 470, 502, 531, 505, 534, 497, 520, 513, 561, 485, 486, 520] payload_rows_per_dst=[5548, 5566, 5488, 5654, 5552, 5588, 5540, 5574] dup_rows_per_dst=[2528, 2658, 2588, 2662, 2694, 2628, 2610, 2658] combine_rows_per_src=[5548, 5566, 5488, 5654, 5552, 5588, 5540, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44340 payload_rows_remote=38698 dup_rows=21196 dup_rows_remote=18502 dedup_ratio=0.323425 router_tokens_per_expert=[502, 486, 529, 516, 534, 499, 560, 503, 486, 538, 507, 492, 473, 507, 502, 515, 551, 495, 505, 499, 536, 480, 469, 529, 491, 540, 490, 527, 529, 530, 500, 512, 516, 513, 513, 536, 522, 530, 490, 483, 528, 478, 501, 480, 530, 522, 506, 527, 504, 520, 519, 495, 509, 518, 528, 527, 513, 505, 512, 498, 514, 512, 531, 556] payload_rows_per_dst=[5642, 5638, 5468, 5496, 5584, 5466, 5546, 5500] dup_rows_per_dst=[2694, 2694, 2582, 2494, 2808, 2634, 2694, 2596] combine_rows_per_src=[5642, 5638, 5468, 5496, 5584, 5466, 5546, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38828 dup_rows=21166 dup_rows_remote=18588 dedup_ratio=0.322968 router_tokens_per_expert=[534, 502, 522, 526, 505, 514, 521, 494, 505, 514, 547, 508, 561, 483, 484, 511, 469, 502, 496, 507, 494, 489, 488, 464, 512, 477, 560, 517, 527, 533, 527, 504, 520, 516, 488, 499, 544, 499, 528, 508, 549, 525, 486, 517, 476, 520, 561, 511, 487, 527, 542, 517, 501, 480, 509, 497, 502, 543, 538, 504, 538, 535, 529, 475] payload_rows_per_dst=[5576, 5514, 5356, 5608, 5544, 5632, 5542, 5598] dup_rows_per_dst=[2660, 2712, 2462, 2706, 2660, 2658, 2578, 2730] combine_rows_per_src=[5576, 5514, 5356, 5608, 5544, 5632, 5542, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44670 payload_rows_remote=38946 dup_rows=20866 dup_rows_remote=17968 dedup_ratio=0.318390 router_tokens_per_expert=[507, 495, 540, 482, 475, 511, 524, 490, 511, 545, 516, 488, 494, 502, 484, 519, 497, 573, 490, 508, 488, 476, 516, 489, 496, 486, 526, 551, 543, 475, 483, 471, 500, 481, 527, 500, 500, 491, 518, 471, 512, 535, 522, 509, 533, 551, 508, 547, 475, 554, 559, 519, 506, 513, 556, 535, 533, 542, 515, 527, 521, 522, 532, 503] payload_rows_per_dst=[5444, 5630, 5534, 5546, 5498, 5594, 5700, 5724] dup_rows_per_dst=[2482, 2610, 2540, 2516, 2478, 2604, 2738, 2898] combine_rows_per_src=[5444, 5630, 5534, 5546, 5498, 5594, 5700, 5724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44544 payload_rows_remote=38918 dup_rows=20992 dup_rows_remote=18338 dedup_ratio=0.320312 router_tokens_per_expert=[500, 483, 505, 544, 506, 497, 522, 502, 491, 519, 523, 507, 502, 499, 529, 559, 525, 477, 539, 508, 529, 514, 481, 517, 463, 482, 496, 510, 493, 519, 530, 491, 501, 528, 558, 557, 497, 500, 523, 476, 511, 513, 503, 494, 508, 494, 509, 559, 532, 520, 509, 527, 513, 505, 537, 503, 534, 497, 486, 525, 568, 484, 506, 529] payload_rows_per_dst=[5526, 5580, 5550, 5498, 5626, 5532, 5636, 5596] dup_rows_per_dst=[2592, 2678, 2630, 2470, 2654, 2650, 2656, 2662] combine_rows_per_src=[5526, 5580, 5550, 5498, 5626, 5532, 5636, 5596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44356 payload_rows_remote=38854 dup_rows=21180 dup_rows_remote=18546 dedup_ratio=0.323181 router_tokens_per_expert=[527, 457, 531, 481, 528, 522, 515, 507, 551, 510, 502, 487, 514, 492, 525, 487, 539, 537, 532, 524, 550, 526, 515, 492, 532, 512, 513, 492, 513, 536, 523, 507, 508, 474, 515, 514, 482, 531, 519, 512, 537, 505, 502, 522, 456, 572, 530, 490, 511, 476, 505, 505, 497, 497, 550, 526, 508, 517, 494, 503, 529, 486, 536, 480] payload_rows_per_dst=[5472, 5502, 5658, 5540, 5570, 5588, 5536, 5490] dup_rows_per_dst=[2664, 2634, 2772, 2716, 2540, 2640, 2598, 2616] combine_rows_per_src=[5472, 5502, 5658, 5540, 5570, 5588, 5536, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44500 payload_rows_remote=38916 dup_rows=21036 dup_rows_remote=18354 dedup_ratio=0.320984 router_tokens_per_expert=[506, 533, 540, 551, 532, 529, 527, 489, 488, 488, 464, 526, 515, 530, 467, 503, 508, 503, 505, 546, 528, 496, 516, 531, 508, 511, 534, 487, 507, 508, 519, 524, 486, 519, 511, 527, 509, 483, 503, 467, 522, 503, 504, 535, 512, 485, 551, 532, 491, 525, 496, 534, 514, 472, 508, 508, 505, 542, 550, 522, 492, 521, 502, 518] payload_rows_per_dst=[5656, 5476, 5584, 5484, 5538, 5618, 5586, 5558] dup_rows_per_dst=[2758, 2486, 2682, 2712, 2472, 2670, 2510, 2746] combine_rows_per_src=[5656, 5476, 5584, 5484, 5538, 5618, 5586, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=228880 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159383056 ingress_bytes=184320000 link_bytes=184320000 token_bytes=318537232 weight_movement_bytes=25165824 total_nvlink_bytes=343703056 weight_to_token_ratio=0.079004 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402462230e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159514624 dispatch_meta_bytes=229680 combine_payload_bytes=159514624 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159744304 ingress_bytes=209846272 link_bytes=209846272 token_bytes=319258928 weight_movement_bytes=50331648 total_nvlink_bytes=369590576 weight_to_token_ratio=0.157651 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596687666e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=158507008 dispatch_meta_bytes=228800 combine_payload_bytes=158507008 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158735808 ingress_bytes=208838656 link_bytes=208838656 token_bytes=317242816 weight_movement_bytes=50331648 total_nvlink_bytes=367574464 weight_to_token_ratio=0.158653 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.589020872e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159039488 dispatch_meta_bytes=229664 combine_payload_bytes=159039488 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159269152 ingress_bytes=234536960 link_bytes=234536960 token_bytes=318308640 weight_movement_bytes=75497472 total_nvlink_bytes=393806112 weight_to_token_ratio=0.237183 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.784555273e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159522816 dispatch_meta_bytes=227656 combine_payload_bytes=159522816 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159750472 ingress_bytes=209854464 link_bytes=209854464 token_bytes=319273288 weight_movement_bytes=50331648 total_nvlink_bytes=369604936 weight_to_token_ratio=0.157644 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596749998e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159408128 dispatch_meta_bytes=229024 combine_payload_bytes=159408128 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159637152 ingress_bytes=209739776 link_bytes=209739776 token_bytes=319045280 weight_movement_bytes=50331648 total_nvlink_bytes=369376928 weight_to_token_ratio=0.157757 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595877354e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159145984 dispatch_meta_bytes=229600 combine_payload_bytes=159145984 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159375584 ingress_bytes=184311808 link_bytes=184311808 token_bytes=318521568 weight_movement_bytes=25165824 total_nvlink_bytes=343687392 weight_to_token_ratio=0.079008 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402399898e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159399936 dispatch_meta_bytes=229080 combine_payload_bytes=159399936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159629016 ingress_bytes=184565760 link_bytes=184565760 token_bytes=319028952 weight_movement_bytes=25165824 total_nvlink_bytes=344194776 weight_to_token_ratio=0.078883 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404332180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44492 payload_rows_remote=38902 dup_rows=21044 dup_rows_remote=18388 dedup_ratio=0.321106 router_tokens_per_expert=[513, 511, 489, 532, 507, 518, 533, 477, 532, 505, 505, 499, 518, 519, 500, 500, 509, 512, 526, 533, 562, 508, 509, 545, 530, 494, 519, 523, 516, 518, 492, 531, 500, 511, 511, 533, 494, 528, 489, 503, 524, 496, 494, 485, 541, 484, 491, 536, 558, 509, 525, 515, 503, 514, 504, 521, 504, 514, 463, 523, 502, 524, 501, 483] payload_rows_per_dst=[5546, 5548, 5658, 5590, 5544, 5584, 5560, 5462] dup_rows_per_dst=[2614, 2608, 2750, 2656, 2594, 2518, 2738, 2566] combine_rows_per_src=[5546, 5548, 5658, 5590, 5544, 5584, 5560, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44506 payload_rows_remote=38882 dup_rows=21030 dup_rows_remote=18424 dedup_ratio=0.320892 router_tokens_per_expert=[528, 471, 521, 551, 541, 533, 523, 507, 553, 561, 516, 503, 520, 472, 478, 483, 521, 512, 531, 516, 480, 503, 478, 505, 504, 480, 489, 508, 542, 493, 527, 518, 487, 550, 529, 494, 516, 495, 535, 512, 522, 484, 518, 501, 522, 494, 534, 540, 521, 533, 513, 521, 487, 555, 500, 506, 493, 533, 510, 525, 495, 494, 494, 487] payload_rows_per_dst=[5690, 5554, 5488, 5504, 5562, 5624, 5572, 5512] dup_rows_per_dst=[2660, 2618, 2604, 2618, 2674, 2606, 2700, 2550] combine_rows_per_src=[5690, 5554, 5488, 5504, 5562, 5624, 5572, 5512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44462 payload_rows_remote=38844 dup_rows=21074 dup_rows_remote=18324 dedup_ratio=0.321564 router_tokens_per_expert=[510, 543, 466, 534, 555, 541, 513, 488, 509, 512, 525, 508, 513, 507, 513, 518, 477, 539, 503, 484, 492, 500, 459, 506, 512, 507, 518, 529, 465, 465, 504, 489, 515, 544, 528, 545, 517, 520, 526, 532, 542, 472, 504, 511, 493, 470, 487, 530, 452, 561, 524, 525, 531, 516, 524, 553, 550, 508, 550, 498, 485, 556, 511, 484] payload_rows_per_dst=[5618, 5548, 5764, 5588, 5608, 5462, 5510, 5364] dup_rows_per_dst=[2750, 2662, 2812, 2644, 2814, 2556, 2572, 2264] combine_rows_per_src=[5618, 5548, 5764, 5588, 5608, 5462, 5510, 5364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159342592 dispatch_meta_bytes=229160 combine_payload_bytes=159342592 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159571752 ingress_bytes=184508416 link_bytes=184508416 token_bytes=318914344 weight_movement_bytes=25165824 total_nvlink_bytes=344080168 weight_to_token_ratio=0.078911 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403895858e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44518 payload_rows_remote=39012 dup_rows=21018 dup_rows_remote=18514 dedup_ratio=0.320709 router_tokens_per_expert=[520, 497, 504, 497, 552, 531, 475, 526, 532, 512, 510, 555, 494, 493, 515, 511, 524, 512, 529, 504, 523, 493, 499, 519, 514, 482, 502, 547, 507, 512, 513, 511, 502, 526, 542, 494, 509, 490, 501, 520, 517, 515, 537, 489, 516, 510, 503, 543, 506, 501, 526, 487, 471, 500, 520, 494, 530, 514, 544, 495, 498, 533, 511, 509] payload_rows_per_dst=[5582, 5502, 5570, 5572, 5568, 5582, 5506, 5636] dup_rows_per_dst=[2622, 2742, 2636, 2604, 2600, 2678, 2504, 2632] combine_rows_per_src=[5582, 5502, 5570, 5572, 5568, 5582, 5506, 5636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44330 payload_rows_remote=38846 dup_rows=21206 dup_rows_remote=18610 dedup_ratio=0.323578 router_tokens_per_expert=[516, 510, 556, 498, 492, 501, 502, 540, 478, 490, 519, 507, 504, 529, 526, 487, 523, 470, 508, 485, 543, 520, 511, 497, 519, 455, 514, 530, 525, 498, 536, 537, 519, 508, 533, 491, 477, 518, 510, 538, 500, 487, 499, 503, 525, 513, 488, 541, 507, 505, 495, 559, 495, 516, 554, 505, 513, 510, 505, 542, 532, 498, 504, 552] payload_rows_per_dst=[5580, 5484, 5492, 5492, 5520, 5560, 5576, 5626] dup_rows_per_dst=[2650, 2596, 2622, 2736, 2668, 2552, 2696, 2686] combine_rows_per_src=[5580, 5484, 5492, 5492, 5520, 5560, 5576, 5626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44372 payload_rows_remote=38848 dup_rows=21164 dup_rows_remote=18472 dedup_ratio=0.322937 router_tokens_per_expert=[535, 501, 511, 513, 498, 550, 514, 512, 487, 530, 507, 529, 477, 490, 485, 562, 518, 485, 500, 487, 515, 508, 511, 512, 497, 530, 477, 538, 521, 527, 513, 469, 514, 498, 493, 538, 521, 516, 509, 519, 545, 494, 466, 541, 533, 523, 475, 511, 511, 499, 533, 532, 498, 475, 521, 525, 525, 518, 512, 527, 473, 527, 542, 545] payload_rows_per_dst=[5594, 5542, 5512, 5580, 5524, 5550, 5456, 5614] dup_rows_per_dst=[2674, 2592, 2560, 2564, 2692, 2626, 2732, 2724] combine_rows_per_src=[5594, 5542, 5512, 5580, 5524, 5550, 5456, 5614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44410 payload_rows_remote=38804 dup_rows=21126 dup_rows_remote=18312 dedup_ratio=0.322357 router_tokens_per_expert=[501, 493, 511, 500, 517, 546, 497, 494, 534, 559, 516, 522, 530, 499, 481, 509, 520, 529, 506, 472, 530, 497, 500, 497, 489, 510, 525, 492, 517, 475, 505, 512, 509, 495, 536, 514, 498, 526, 520, 515, 494, 503, 501, 527, 498, 511, 495, 521, 522, 503, 520, 521, 516, 491, 541, 510, 547, 484, 547, 542, 515, 547, 531, 483] payload_rows_per_dst=[5396, 5556, 5658, 5418, 5610, 5562, 5604, 5606] dup_rows_per_dst=[2408, 2790, 2712, 2632, 2616, 2538, 2616, 2814] combine_rows_per_src=[5396, 5556, 5658, 5418, 5610, 5562, 5604, 5606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159260672 dispatch_meta_bytes=229224 combine_payload_bytes=159260672 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159489896 ingress_bytes=184426496 link_bytes=184426496 token_bytes=318750568 weight_movement_bytes=25165824 total_nvlink_bytes=343916392 weight_to_token_ratio=0.078951 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403272542e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44448 payload_rows_remote=38886 dup_rows=21088 dup_rows_remote=18464 dedup_ratio=0.321777 router_tokens_per_expert=[478, 527, 507, 524, 518, 484, 538, 524, 532, 526, 516, 503, 525, 499, 525, 517, 481, 521, 516, 546, 528, 500, 512, 489, 501, 508, 531, 501, 515, 495, 503, 530, 496, 536, 511, 520, 486, 530, 460, 529, 512, 491, 522, 482, 506, 479, 500, 494, 530, 543, 504, 498, 504, 492, 495, 517, 542, 513, 516, 539, 545, 534, 530, 492] payload_rows_per_dst=[5556, 5566, 5562, 5530, 5506, 5516, 5488, 5724] dup_rows_per_dst=[2644, 2720, 2624, 2638, 2630, 2456, 2678, 2698] combine_rows_per_src=[5556, 5566, 5562, 5530, 5506, 5516, 5488, 5724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=228672 combine_payload_bytes=159105024 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159333696 ingress_bytes=209436672 link_bytes=209436672 token_bytes=318438720 weight_movement_bytes=50331648 total_nvlink_bytes=368770368 weight_to_token_ratio=0.158058 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593571083e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=229824 combine_payload_bytes=159113216 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159343040 ingress_bytes=184279040 link_bytes=184279040 token_bytes=318456256 weight_movement_bytes=25165824 total_nvlink_bytes=343622080 weight_to_token_ratio=0.079024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402150572e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159793152 dispatch_meta_bytes=230104 combine_payload_bytes=159793152 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=160023256 ingress_bytes=235290624 link_bytes=235290624 token_bytes=319816408 weight_movement_bytes=75497472 total_nvlink_bytes=395313880 weight_to_token_ratio=0.236065 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790289785e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159121408 dispatch_meta_bytes=229280 combine_payload_bytes=159121408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159350688 ingress_bytes=184287232 link_bytes=184287232 token_bytes=318472096 weight_movement_bytes=25165824 total_nvlink_bytes=343637920 weight_to_token_ratio=0.079020 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402212903e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=158941184 dispatch_meta_bytes=228464 combine_payload_bytes=158941184 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159169648 ingress_bytes=234438656 link_bytes=234438656 token_bytes=318110832 weight_movement_bytes=75497472 total_nvlink_bytes=393608304 weight_to_token_ratio=0.237331 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783807293e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=229400 combine_payload_bytes=159277056 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159506456 ingress_bytes=184442880 link_bytes=184442880 token_bytes=318783512 weight_movement_bytes=25165824 total_nvlink_bytes=343949336 weight_to_token_ratio=0.078943 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403397205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44518 payload_rows_remote=38956 dup_rows=21018 dup_rows_remote=18512 dedup_ratio=0.320709 router_tokens_per_expert=[523, 522, 506, 521, 482, 494, 514, 522, 506, 524, 527, 446, 543, 508, 538, 548, 518, 493, 519, 522, 518, 545, 553, 548, 531, 491, 526, 515, 518, 503, 480, 470, 502, 535, 485, 525, 512, 508, 461, 481, 504, 531, 521, 492, 470, 514, 492, 500, 553, 495, 509, 527, 538, 467, 520, 536, 488, 543, 501, 511, 518, 493, 515, 547] payload_rows_per_dst=[5538, 5602, 5578, 5562, 5508, 5554, 5650, 5526] dup_rows_per_dst=[2630, 2678, 2854, 2506, 2510, 2494, 2640, 2706] combine_rows_per_src=[5538, 5602, 5578, 5562, 5508, 5554, 5650, 5526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44594 payload_rows_remote=39024 dup_rows=20942 dup_rows_remote=18262 dedup_ratio=0.319550 router_tokens_per_expert=[472, 509, 510, 552, 534, 496, 518, 518, 521, 519, 472, 532, 506, 501, 530, 528, 479, 548, 557, 511, 495, 526, 496, 489, 516, 487, 516, 507, 493, 531, 468, 511, 502, 486, 519, 490, 487, 501, 521, 521, 520, 497, 518, 523, 465, 553, 527, 522, 525, 519, 497, 521, 503, 552, 546, 500, 496, 527, 501, 530, 527, 535, 489, 500] payload_rows_per_dst=[5574, 5602, 5592, 5518, 5552, 5570, 5648, 5538] dup_rows_per_dst=[2644, 2616, 2610, 2540, 2502, 2680, 2678, 2672] combine_rows_per_src=[5574, 5602, 5592, 5518, 5552, 5570, 5648, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159563776 dispatch_meta_bytes=229872 combine_payload_bytes=159563776 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159793648 ingress_bytes=184729600 link_bytes=184729600 token_bytes=319357424 weight_movement_bytes=25165824 total_nvlink_bytes=344523248 weight_to_token_ratio=0.078801 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405578813e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44554 payload_rows_remote=38876 dup_rows=20982 dup_rows_remote=18340 dedup_ratio=0.320160 router_tokens_per_expert=[538, 516, 497, 525, 506, 555, 495, 527, 499, 508, 512, 538, 498, 497, 512, 495, 506, 512, 496, 470, 549, 487, 505, 519, 511, 508, 517, 522, 521, 493, 519, 511, 485, 509, 487, 519, 478, 496, 491, 489, 549, 491, 488, 497, 492, 526, 506, 498, 515, 543, 552, 528, 516, 539, 504, 519, 527, 513, 541, 508, 495, 488, 556, 559] payload_rows_per_dst=[5678, 5526, 5560, 5630, 5584, 5660, 5544, 5372] dup_rows_per_dst=[2642, 2592, 2528, 2772, 2644, 2924, 2566, 2314] combine_rows_per_src=[5678, 5526, 5560, 5630, 5584, 5660, 5544, 5372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44564 payload_rows_remote=38906 dup_rows=20972 dup_rows_remote=18386 dedup_ratio=0.320007 router_tokens_per_expert=[506, 525, 488, 508, 513, 496, 515, 518, 488, 542, 490, 498, 524, 541, 498, 541, 523, 491, 508, 545, 501, 505, 512, 506, 518, 550, 510, 515, 526, 546, 490, 505, 595, 465, 520, 506, 490, 516, 489, 490, 550, 493, 517, 541, 490, 512, 519, 497, 481, 515, 501, 517, 491, 504, 496, 502, 517, 539, 523, 517, 524, 496, 474, 539] payload_rows_per_dst=[5500, 5658, 5532, 5510, 5558, 5610, 5568, 5628] dup_rows_per_dst=[2638, 2586, 2650, 2810, 2584, 2628, 2446, 2630] combine_rows_per_src=[5500, 5658, 5532, 5510, 5558, 5610, 5568, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44418 payload_rows_remote=38844 dup_rows=21118 dup_rows_remote=18372 dedup_ratio=0.322235 router_tokens_per_expert=[530, 462, 501, 529, 536, 518, 523, 490, 482, 476, 494, 514, 497, 534, 482, 515, 530, 493, 526, 536, 515, 502, 538, 507, 487, 522, 497, 543, 474, 533, 493, 505, 533, 507, 527, 509, 514, 512, 545, 513, 512, 506, 503, 486, 511, 500, 523, 549, 500, 522, 517, 490, 510, 552, 543, 535, 524, 502, 503, 499, 533, 493, 516, 495] payload_rows_per_dst=[5568, 5460, 5612, 5554, 5574, 5518, 5622, 5510] dup_rows_per_dst=[2610, 2528, 2682, 2554, 2746, 2662, 2716, 2620] combine_rows_per_src=[5568, 5460, 5612, 5554, 5574, 5518, 5622, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44546 payload_rows_remote=39000 dup_rows=20990 dup_rows_remote=18442 dedup_ratio=0.320282 router_tokens_per_expert=[530, 527, 467, 528, 526, 503, 512, 539, 492, 517, 497, 526, 475, 549, 535, 536, 534, 502, 521, 499, 499, 529, 495, 504, 502, 490, 545, 501, 501, 469, 515, 490, 467, 538, 519, 510, 511, 524, 536, 508, 525, 506, 540, 512, 491, 500, 504, 525, 511, 514, 512, 491, 502, 521, 518, 478, 517, 510, 536, 544, 535, 491, 496, 521] payload_rows_per_dst=[5582, 5620, 5520, 5482, 5614, 5534, 5546, 5648] dup_rows_per_dst=[2682, 2634, 2646, 2544, 2612, 2672, 2548, 2652] combine_rows_per_src=[5582, 5620, 5520, 5482, 5614, 5534, 5546, 5648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159842304 dispatch_meta_bytes=229144 combine_payload_bytes=159842304 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=160071448 ingress_bytes=235339776 link_bytes=235339776 token_bytes=319913752 weight_movement_bytes=75497472 total_nvlink_bytes=395411224 weight_to_token_ratio=0.235993 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.790663775e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44464 payload_rows_remote=39042 dup_rows=21072 dup_rows_remote=18380 dedup_ratio=0.321533 router_tokens_per_expert=[539, 541, 473, 526, 510, 497, 521, 491, 501, 482, 494, 498, 512, 472, 537, 490, 545, 519, 554, 497, 533, 589, 520, 536, 487, 509, 494, 540, 535, 472, 515, 496, 527, 529, 525, 506, 500, 520, 476, 489, 518, 493, 510, 504, 507, 518, 529, 510, 518, 533, 543, 504, 553, 472, 515, 487, 500, 500, 529, 492, 510, 519, 501, 506] payload_rows_per_dst=[5546, 5522, 5558, 5662, 5672, 5532, 5550, 5422] dup_rows_per_dst=[2650, 2450, 2468, 2690, 2776, 2646, 2700, 2692] combine_rows_per_src=[5546, 5522, 5558, 5662, 5672, 5532, 5550, 5422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44294 payload_rows_remote=38782 dup_rows=21242 dup_rows_remote=18698 dedup_ratio=0.324127 router_tokens_per_expert=[513, 506, 530, 539, 523, 486, 526, 489, 490, 566, 522, 515, 516, 511, 487, 494, 499, 517, 477, 521, 498, 499, 536, 481, 502, 470, 519, 539, 519, 506, 530, 522, 495, 474, 538, 506, 529, 530, 499, 528, 542, 532, 515, 520, 551, 511, 510, 539, 504, 481, 499, 492, 533, 516, 515, 506, 507, 515, 460, 520, 501, 527, 548, 477] payload_rows_per_dst=[5554, 5506, 5512, 5552, 5512, 5674, 5488, 5496] dup_rows_per_dst=[2670, 2696, 2544, 2662, 2686, 2766, 2604, 2614] combine_rows_per_src=[5554, 5506, 5512, 5552, 5512, 5674, 5488, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=159236096 dispatch_meta_bytes=228864 combine_payload_bytes=159236096 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159464960 ingress_bytes=234733568 link_bytes=234733568 token_bytes=318701056 weight_movement_bytes=75497472 total_nvlink_bytes=394198528 weight_to_token_ratio=0.236891 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786051233e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229168 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159588144 ingress_bytes=184524800 link_bytes=184524800 token_bytes=318947120 weight_movement_bytes=25165824 total_nvlink_bytes=344112944 weight_to_token_ratio=0.078903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404020521e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159105024 dispatch_meta_bytes=228864 combine_payload_bytes=159105024 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159333888 ingress_bytes=184270848 link_bytes=184270848 token_bytes=318438912 weight_movement_bytes=25165824 total_nvlink_bytes=343604736 weight_to_token_ratio=0.079029 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402088240e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159744000 dispatch_meta_bytes=229768 combine_payload_bytes=159744000 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159973768 ingress_bytes=210075648 link_bytes=210075648 token_bytes=319717768 weight_movement_bytes=50331648 total_nvlink_bytes=370049416 weight_to_token_ratio=0.157425 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598432952e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44516 payload_rows_remote=38942 dup_rows=21020 dup_rows_remote=18346 dedup_ratio=0.320740 router_tokens_per_expert=[523, 526, 479, 541, 497, 494, 508, 483, 497, 505, 528, 505, 528, 526, 520, 485, 477, 536, 482, 501, 503, 548, 532, 506, 515, 525, 511, 522, 518, 517, 512, 504, 509, 552, 505, 498, 495, 537, 480, 544, 526, 515, 527, 506, 530, 508, 521, 495, 471, 492, 518, 511, 517, 505, 518, 537, 556, 512, 519, 510, 525, 485, 477, 513] payload_rows_per_dst=[5494, 5564, 5512, 5574, 5584, 5652, 5538, 5598] dup_rows_per_dst=[2608, 2624, 2658, 2674, 2656, 2604, 2600, 2596] combine_rows_per_src=[5494, 5564, 5512, 5574, 5584, 5652, 5538, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159916032 dispatch_meta_bytes=229688 combine_payload_bytes=159916032 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160145720 ingress_bytes=210247680 link_bytes=210247680 token_bytes=320061752 weight_movement_bytes=50331648 total_nvlink_bytes=370393400 weight_to_token_ratio=0.157256 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599741917e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=158851072 dispatch_meta_bytes=229920 combine_payload_bytes=158851072 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159080992 ingress_bytes=234348544 link_bytes=234348544 token_bytes=317932064 weight_movement_bytes=75497472 total_nvlink_bytes=393429536 weight_to_token_ratio=0.237464 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783121645e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=229152 combine_payload_bytes=159506432 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159735584 ingress_bytes=235003904 link_bytes=235003904 token_bytes=319242016 weight_movement_bytes=75497472 total_nvlink_bytes=394739488 weight_to_token_ratio=0.236490 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788108177e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44384 payload_rows_remote=38766 dup_rows=21152 dup_rows_remote=18354 dedup_ratio=0.322754 router_tokens_per_expert=[524, 495, 523, 482, 550, 506, 523, 525, 482, 497, 517, 503, 475, 542, 460, 508, 505, 502, 533, 564, 533, 479, 483, 496, 493, 524, 517, 521, 546, 492, 505, 520, 491, 506, 492, 547, 485, 526, 541, 515, 513, 534, 550, 522, 537, 532, 487, 533, 547, 494, 501, 500, 528, 540, 508, 485, 491, 499, 469, 514, 482, 533, 502, 539] payload_rows_per_dst=[5532, 5442, 5552, 5584, 5586, 5618, 5498, 5572] dup_rows_per_dst=[2724, 2526, 2638, 2652, 2620, 2798, 2708, 2486] combine_rows_per_src=[5532, 5442, 5552, 5584, 5586, 5618, 5498, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44810 payload_rows_remote=39158 dup_rows=20726 dup_rows_remote=18148 dedup_ratio=0.316254 router_tokens_per_expert=[528, 553, 511, 491, 506, 559, 485, 529, 535, 501, 515, 530, 496, 538, 504, 496, 508, 490, 471, 486, 517, 513, 479, 491, 495, 540, 527, 509, 513, 525, 489, 542, 486, 490, 517, 514, 520, 508, 515, 522, 500, 489, 565, 539, 506, 564, 511, 499, 524, 481, 482, 474, 537, 531, 507, 515, 502, 510, 549, 508, 536, 525, 489, 481] payload_rows_per_dst=[5624, 5652, 5462, 5636, 5606, 5640, 5632, 5558] dup_rows_per_dst=[2700, 2578, 2448, 2644, 2538, 2706, 2470, 2642] combine_rows_per_src=[5624, 5652, 5462, 5636, 5606, 5640, 5632, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44528 payload_rows_remote=38974 dup_rows=21008 dup_rows_remote=18374 dedup_ratio=0.320557 router_tokens_per_expert=[514, 488, 503, 497, 506, 542, 507, 485, 510, 548, 517, 484, 488, 509, 513, 502, 496, 505, 541, 487, 523, 520, 497, 515, 509, 502, 510, 529, 498, 505, 549, 541, 534, 494, 483, 528, 507, 507, 527, 514, 489, 490, 535, 497, 496, 494, 490, 465, 528, 535, 497, 561, 527, 529, 487, 506, 543, 524, 540, 506, 522, 525, 510, 538] payload_rows_per_dst=[5546, 5550, 5580, 5574, 5554, 5434, 5606, 5684] dup_rows_per_dst=[2538, 2592, 2588, 2712, 2634, 2478, 2734, 2732] combine_rows_per_src=[5546, 5550, 5580, 5574, 5554, 5434, 5606, 5684] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44536 payload_rows_remote=38982 dup_rows=21000 dup_rows_remote=18404 dedup_ratio=0.320435 router_tokens_per_expert=[516, 492, 473, 504, 514, 478, 515, 507, 497, 532, 502, 517, 522, 538, 507, 550, 484, 546, 527, 539, 512, 486, 474, 498, 526, 521, 509, 541, 482, 548, 515, 553, 523, 473, 511, 507, 499, 469, 541, 542, 539, 528, 474, 535, 517, 522, 488, 494, 519, 535, 517, 502, 547, 508, 486, 470, 510, 539, 507, 485, 515, 517, 480, 544] payload_rows_per_dst=[5554, 5594, 5508, 5652, 5542, 5518, 5616, 5552] dup_rows_per_dst=[2596, 2784, 2670, 2738, 2588, 2478, 2516, 2630] combine_rows_per_src=[5554, 5594, 5508, 5652, 5542, 5518, 5616, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44436 payload_rows_remote=38826 dup_rows=21100 dup_rows_remote=18546 dedup_ratio=0.321960 router_tokens_per_expert=[491, 518, 491, 512, 487, 551, 527, 509, 512, 544, 556, 503, 471, 543, 476, 533, 506, 503, 535, 554, 529, 492, 511, 500, 514, 498, 529, 490, 521, 506, 498, 504, 480, 498, 535, 523, 589, 505, 497, 489, 501, 498, 516, 516, 540, 517, 500, 487, 505, 520, 495, 494, 544, 526, 514, 484, 500, 533, 496, 479, 491, 545, 548, 489] payload_rows_per_dst=[5550, 5614, 5606, 5542, 5500, 5524, 5610, 5490] dup_rows_per_dst=[2622, 2662, 2654, 2578, 2732, 2626, 2554, 2672] combine_rows_per_src=[5550, 5614, 5606, 5542, 5500, 5524, 5610, 5490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158785536 dispatch_meta_bytes=228480 combine_payload_bytes=158785536 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159014016 ingress_bytes=183951360 link_bytes=183951360 token_bytes=317799552 weight_movement_bytes=25165824 total_nvlink_bytes=342965376 weight_to_token_ratio=0.079188 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399657306e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44430 payload_rows_remote=38774 dup_rows=21106 dup_rows_remote=18354 dedup_ratio=0.322052 router_tokens_per_expert=[513, 505, 510, 539, 509, 521, 525, 462, 513, 522, 539, 514, 525, 504, 532, 497, 533, 486, 473, 460, 502, 512, 499, 520, 532, 516, 505, 485, 503, 500, 537, 482, 513, 524, 514, 551, 526, 487, 470, 537, 545, 540, 493, 532, 487, 478, 512, 529, 491, 519, 525, 543, 549, 540, 520, 508, 480, 507, 508, 517, 506, 523, 531, 488] payload_rows_per_dst=[5490, 5604, 5490, 5460, 5420, 5574, 5736, 5656] dup_rows_per_dst=[2576, 2688, 2480, 2648, 2536, 2670, 2756, 2752] combine_rows_per_src=[5490, 5604, 5490, 5460, 5420, 5574, 5736, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44394 payload_rows_remote=38874 dup_rows=21142 dup_rows_remote=18574 dedup_ratio=0.322601 router_tokens_per_expert=[519, 551, 514, 483, 506, 524, 505, 539, 510, 507, 533, 550, 545, 523, 500, 509, 511, 509, 528, 502, 490, 483, 521, 500, 524, 512, 512, 521, 513, 474, 490, 562, 499, 506, 465, 553, 530, 529, 473, 513, 501, 512, 480, 537, 457, 527, 538, 490, 508, 525, 518, 527, 499, 497, 507, 499, 499, 529, 508, 492, 536, 503, 521, 520] payload_rows_per_dst=[5634, 5666, 5520, 5520, 5486, 5492, 5524, 5552] dup_rows_per_dst=[2648, 2688, 2568, 2696, 2650, 2592, 2636, 2664] combine_rows_per_src=[5634, 5666, 5520, 5520, 5486, 5492, 5524, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=160391168 dispatch_meta_bytes=229224 combine_payload_bytes=160391168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160620392 ingress_bytes=185556992 link_bytes=185556992 token_bytes=321011560 weight_movement_bytes=25165824 total_nvlink_bytes=346177384 weight_to_token_ratio=0.078395 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.411874310e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159637504 dispatch_meta_bytes=229392 combine_payload_bytes=159637504 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159866896 ingress_bytes=209969152 link_bytes=209969152 token_bytes=319504400 weight_movement_bytes=50331648 total_nvlink_bytes=369836048 weight_to_token_ratio=0.157530 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.597622641e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=159670272 dispatch_meta_bytes=229544 combine_payload_bytes=159670272 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159899816 ingress_bytes=184836096 link_bytes=184836096 token_bytes=319570088 weight_movement_bytes=25165824 total_nvlink_bytes=344735912 weight_to_token_ratio=0.078749 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406389124e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44596 payload_rows_remote=39018 dup_rows=20940 dup_rows_remote=18342 dedup_ratio=0.319519 router_tokens_per_expert=[514, 508, 510, 519, 498, 487, 484, 530, 481, 505, 495, 511, 528, 486, 524, 501, 484, 519, 553, 508, 543, 495, 522, 499, 509, 485, 471, 571, 515, 517, 501, 519, 516, 514, 546, 526, 497, 512, 484, 487, 531, 522, 497, 488, 522, 525, 568, 522, 492, 521, 510, 486, 523, 522, 527, 512, 527, 493, 515, 534, 521, 493, 493, 550] payload_rows_per_dst=[5490, 5524, 5642, 5578, 5528, 5598, 5558, 5678] dup_rows_per_dst=[2610, 2538, 2604, 2598, 2636, 2752, 2628, 2574] combine_rows_per_src=[5490, 5524, 5642, 5578, 5528, 5598, 5558, 5678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159031296 dispatch_meta_bytes=229488 combine_payload_bytes=159031296 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159260784 ingress_bytes=209362944 link_bytes=209362944 token_bytes=318292080 weight_movement_bytes=50331648 total_nvlink_bytes=368623728 weight_to_token_ratio=0.158130 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593010098e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=158818304 dispatch_meta_bytes=228512 combine_payload_bytes=158818304 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159046816 ingress_bytes=209149952 link_bytes=209149952 token_bytes=317865120 weight_movement_bytes=50331648 total_nvlink_bytes=368196768 weight_to_token_ratio=0.158343 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591389475e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159227904 dispatch_meta_bytes=229792 combine_payload_bytes=159227904 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159457696 ingress_bytes=184393728 link_bytes=184393728 token_bytes=318685600 weight_movement_bytes=25165824 total_nvlink_bytes=343851424 weight_to_token_ratio=0.078968 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403023215e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159817728 dispatch_meta_bytes=229440 combine_payload_bytes=159817728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160047168 ingress_bytes=184983552 link_bytes=184983552 token_bytes=319864896 weight_movement_bytes=25165824 total_nvlink_bytes=345030720 weight_to_token_ratio=0.078676 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407511094e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44408 payload_rows_remote=38856 dup_rows=21128 dup_rows_remote=18424 dedup_ratio=0.322388 router_tokens_per_expert=[558, 503, 507, 498, 501, 491, 533, 479, 524, 492, 511, 477, 524, 532, 521, 521, 495, 548, 486, 540, 529, 524, 487, 504, 464, 475, 568, 498, 528, 500, 518, 534, 537, 508, 525, 542, 511, 506, 486, 518, 525, 515, 514, 549, 510, 512, 502, 501, 526, 524, 505, 493, 504, 513, 495, 544, 478, 512, 498, 509, 514, 518, 489, 515] payload_rows_per_dst=[5516, 5574, 5554, 5536, 5616, 5552, 5506, 5554] dup_rows_per_dst=[2624, 2630, 2672, 2634, 2650, 2704, 2702, 2512] combine_rows_per_src=[5516, 5574, 5554, 5536, 5616, 5552, 5506, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44800 payload_rows_remote=39246 dup_rows=20736 dup_rows_remote=18100 dedup_ratio=0.316406 router_tokens_per_expert=[511, 522, 537, 510, 502, 501, 487, 512, 516, 509, 526, 543, 493, 501, 502, 505, 498, 526, 537, 497, 534, 556, 528, 504, 489, 491, 523, 512, 480, 514, 513, 484, 540, 507, 517, 530, 497, 521, 494, 509, 503, 560, 508, 559, 474, 522, 498, 472, 533, 504, 523, 528, 517, 514, 479, 489, 511, 540, 521, 495, 511, 521, 497, 511] payload_rows_per_dst=[5548, 5554, 5714, 5560, 5634, 5580, 5610, 5600] dup_rows_per_dst=[2616, 2636, 2646, 2452, 2596, 2612, 2564, 2614] combine_rows_per_src=[5548, 5554, 5714, 5560, 5634, 5580, 5610, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44490 payload_rows_remote=38926 dup_rows=21046 dup_rows_remote=18382 dedup_ratio=0.321136 router_tokens_per_expert=[498, 509, 532, 508, 518, 488, 525, 511, 503, 499, 474, 470, 519, 539, 511, 544, 521, 500, 547, 545, 513, 482, 497, 515, 525, 510, 535, 491, 487, 513, 502, 499, 503, 501, 568, 483, 515, 528, 515, 501, 510, 519, 466, 503, 499, 513, 467, 518, 507, 549, 524, 495, 483, 557, 510, 524, 530, 548, 496, 536, 518, 519, 498, 535] payload_rows_per_dst=[5574, 5530, 5670, 5566, 5564, 5412, 5588, 5586] dup_rows_per_dst=[2604, 2588, 2570, 2558, 2664, 2578, 2710, 2774] combine_rows_per_src=[5574, 5530, 5670, 5566, 5564, 5412, 5588, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44446 payload_rows_remote=38878 dup_rows=21090 dup_rows_remote=18408 dedup_ratio=0.321808 router_tokens_per_expert=[519, 503, 533, 515, 522, 499, 503, 523, 507, 481, 480, 501, 546, 525, 484, 502, 552, 555, 492, 514, 501, 539, 544, 482, 463, 530, 516, 499, 502, 524, 486, 526, 486, 543, 516, 508, 517, 503, 506, 541, 494, 504, 544, 491, 505, 497, 513, 516, 484, 474, 509, 541, 509, 516, 511, 520, 522, 541, 502, 498, 519, 538, 514, 518] payload_rows_per_dst=[5568, 5476, 5748, 5558, 5618, 5526, 5422, 5530] dup_rows_per_dst=[2682, 2610, 3032, 2484, 2816, 2602, 2512, 2352] combine_rows_per_src=[5568, 5476, 5748, 5558, 5618, 5526, 5422, 5530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229120 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159383296 ingress_bytes=209485824 link_bytes=209485824 token_bytes=318537472 weight_movement_bytes=50331648 total_nvlink_bytes=368869120 weight_to_token_ratio=0.158009 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593945073e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44370 payload_rows_remote=38762 dup_rows=21166 dup_rows_remote=18460 dedup_ratio=0.322968 router_tokens_per_expert=[519, 556, 523, 498, 490, 530, 524, 484, 493, 509, 504, 508, 488, 532, 508, 522, 530, 537, 495, 526, 505, 554, 525, 500, 476, 481, 478, 515, 518, 490, 535, 509, 524, 528, 521, 524, 531, 518, 510, 490, 561, 539, 526, 484, 485, 513, 460, 521, 523, 500, 511, 570, 474, 480, 504, 548, 517, 484, 515, 508, 512, 497, 507, 521] payload_rows_per_dst=[5522, 5496, 5202, 5844, 5674, 5498, 5526, 5608] dup_rows_per_dst=[2642, 2508, 2270, 3032, 2764, 2540, 2704, 2706] combine_rows_per_src=[5522, 5496, 5202, 5844, 5674, 5498, 5526, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38930 dup_rows=21122 dup_rows_remote=18484 dedup_ratio=0.322296 router_tokens_per_expert=[514, 523, 504, 501, 537, 513, 536, 518, 515, 473, 507, 543, 457, 545, 566, 515, 505, 502, 519, 501, 510, 519, 529, 488, 542, 531, 496, 486, 547, 495, 476, 466, 492, 484, 508, 518, 499, 460, 538, 538, 530, 519, 501, 511, 527, 507, 531, 509, 519, 495, 533, 488, 504, 496, 513, 513, 529, 521, 507, 561, 517, 514, 491, 516] payload_rows_per_dst=[5590, 5612, 5512, 5504, 5528, 5574, 5484, 5610] dup_rows_per_dst=[2702, 2630, 2634, 2574, 2546, 2696, 2638, 2702] combine_rows_per_src=[5590, 5612, 5512, 5504, 5528, 5574, 5484, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44602 payload_rows_remote=39024 dup_rows=20934 dup_rows_remote=18360 dedup_ratio=0.319427 router_tokens_per_expert=[538, 513, 524, 488, 490, 530, 521, 512, 503, 499, 497, 525, 510, 528, 505, 561, 524, 486, 500, 517, 496, 508, 494, 551, 535, 510, 511, 544, 499, 509, 525, 449, 523, 485, 545, 490, 512, 514, 505, 462, 550, 504, 543, 486, 524, 500, 513, 477, 531, 526, 508, 525, 509, 508, 484, 526, 545, 509, 516, 490, 547, 500, 493, 516] payload_rows_per_dst=[5570, 5596, 5578, 5556, 5516, 5566, 5598, 5622] dup_rows_per_dst=[2662, 2660, 2574, 2608, 2556, 2628, 2636, 2610] combine_rows_per_src=[5570, 5596, 5578, 5556, 5516, 5566, 5598, 5622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=160751616 dispatch_meta_bytes=229384 combine_payload_bytes=160751616 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160981000 ingress_bytes=211083264 link_bytes=211083264 token_bytes=321732616 weight_movement_bytes=50331648 total_nvlink_bytes=372064264 weight_to_token_ratio=0.156439 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.606099746e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159440896 dispatch_meta_bytes=229232 combine_payload_bytes=159440896 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159670128 ingress_bytes=184606720 link_bytes=184606720 token_bytes=319111024 weight_movement_bytes=25165824 total_nvlink_bytes=344276848 weight_to_token_ratio=0.078862 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404643838e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44664 payload_rows_remote=39048 dup_rows=20872 dup_rows_remote=18240 dedup_ratio=0.318481 router_tokens_per_expert=[508, 505, 478, 520, 540, 523, 521, 526, 498, 512, 511, 527, 517, 548, 528, 498, 495, 485, 482, 477, 535, 525, 520, 523, 504, 535, 517, 534, 520, 521, 514, 479, 509, 511, 512, 504, 495, 500, 504, 508, 515, 474, 500, 544, 483, 534, 541, 535, 493, 519, 488, 513, 509, 520, 513, 528, 524, 489, 492, 516, 459, 562, 519, 529] payload_rows_per_dst=[5652, 5622, 5536, 5616, 5504, 5610, 5564, 5560] dup_rows_per_dst=[2590, 2656, 2548, 2632, 2582, 2642, 2602, 2620] combine_rows_per_src=[5652, 5622, 5536, 5616, 5504, 5610, 5564, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158769152 dispatch_meta_bytes=228888 combine_payload_bytes=158769152 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158998040 ingress_bytes=209100800 link_bytes=209100800 token_bytes=317767192 weight_movement_bytes=50331648 total_nvlink_bytes=368098840 weight_to_token_ratio=0.158392 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.591015485e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=229144 combine_payload_bytes=159244288 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159473432 ingress_bytes=209575936 link_bytes=209575936 token_bytes=318717720 weight_movement_bytes=50331648 total_nvlink_bytes=369049368 weight_to_token_ratio=0.157919 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594630721e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159457280 dispatch_meta_bytes=229656 combine_payload_bytes=159457280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159686936 ingress_bytes=184623104 link_bytes=184623104 token_bytes=319144216 weight_movement_bytes=25165824 total_nvlink_bytes=344310040 weight_to_token_ratio=0.078854 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404768501e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=159842304 dispatch_meta_bytes=229536 combine_payload_bytes=159842304 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160071840 ingress_bytes=210173952 link_bytes=210173952 token_bytes=319914144 weight_movement_bytes=50331648 total_nvlink_bytes=370245792 weight_to_token_ratio=0.157329 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.599180932e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159940608 dispatch_meta_bytes=229152 combine_payload_bytes=159940608 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160169760 ingress_bytes=185106432 link_bytes=185106432 token_bytes=320110368 weight_movement_bytes=25165824 total_nvlink_bytes=345276192 weight_to_token_ratio=0.078616 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408446069e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44512 payload_rows_remote=38932 dup_rows=21024 dup_rows_remote=18356 dedup_ratio=0.320801 router_tokens_per_expert=[551, 541, 521, 541, 495, 495, 522, 507, 537, 527, 510, 476, 499, 525, 504, 499, 553, 517, 496, 531, 475, 516, 489, 533, 489, 508, 499, 539, 509, 510, 515, 492, 508, 501, 515, 499, 506, 515, 521, 519, 552, 503, 519, 534, 495, 510, 513, 498, 513, 466, 474, 545, 514, 503, 515, 507, 512, 520, 511, 511, 528, 516, 507, 497] payload_rows_per_dst=[5640, 5582, 5540, 5554, 5572, 5580, 5474, 5570] dup_rows_per_dst=[2706, 2572, 2680, 2568, 2596, 2668, 2600, 2634] combine_rows_per_src=[5640, 5582, 5540, 5554, 5572, 5580, 5474, 5570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44620 payload_rows_remote=39076 dup_rows=20916 dup_rows_remote=18370 dedup_ratio=0.319153 router_tokens_per_expert=[458, 504, 525, 528, 478, 550, 465, 511, 479, 504, 521, 484, 535, 477, 506, 539, 521, 492, 513, 528, 505, 524, 562, 483, 487, 485, 490, 541, 549, 527, 564, 514, 483, 491, 548, 547, 527, 504, 515, 536, 522, 465, 520, 534, 508, 498, 533, 511, 525, 497, 499, 470, 483, 512, 481, 502, 530, 508, 510, 505, 549, 560, 510, 536] payload_rows_per_dst=[5520, 5544, 5648, 5630, 5576, 5524, 5464, 5714] dup_rows_per_dst=[2518, 2546, 2608, 2684, 2726, 2658, 2474, 2702] combine_rows_per_src=[5520, 5544, 5648, 5630, 5576, 5524, 5464, 5714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159465472 dispatch_meta_bytes=229152 combine_payload_bytes=159465472 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159694624 ingress_bytes=209797120 link_bytes=209797120 token_bytes=319160096 weight_movement_bytes=50331648 total_nvlink_bytes=369491744 weight_to_token_ratio=0.157700 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596313676e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44728 payload_rows_remote=39060 dup_rows=20808 dup_rows_remote=18102 dedup_ratio=0.317505 router_tokens_per_expert=[497, 489, 522, 494, 524, 549, 521, 511, 540, 524, 502, 526, 475, 497, 520, 485, 508, 531, 504, 475, 517, 539, 535, 530, 476, 521, 508, 495, 533, 532, 509, 494, 536, 527, 518, 547, 482, 558, 507, 512, 483, 501, 503, 514, 499, 509, 490, 542, 509, 524, 488, 509, 531, 489, 482, 531, 517, 542, 532, 485, 520, 498, 516, 484] payload_rows_per_dst=[5594, 5600, 5624, 5576, 5668, 5502, 5616, 5548] dup_rows_per_dst=[2620, 2538, 2654, 2560, 2706, 2580, 2510, 2640] combine_rows_per_src=[5594, 5600, 5624, 5576, 5668, 5502, 5616, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44488 payload_rows_remote=38928 dup_rows=21048 dup_rows_remote=18352 dedup_ratio=0.321167 router_tokens_per_expert=[537, 494, 514, 531, 521, 512, 545, 477, 505, 496, 502, 529, 534, 513, 521, 499, 551, 491, 482, 523, 498, 522, 511, 503, 484, 495, 502, 522, 499, 489, 518, 480, 537, 495, 522, 513, 506, 513, 488, 528, 493, 524, 545, 477, 529, 510, 506, 510, 507, 486, 496, 536, 529, 509, 548, 543, 511, 511, 531, 513, 509, 492, 503, 548] payload_rows_per_dst=[5362, 5572, 5484, 5682, 5578, 5618, 5632, 5560] dup_rows_per_dst=[2400, 2626, 2542, 2796, 2712, 2582, 2694, 2696] combine_rows_per_src=[5362, 5572, 5484, 5682, 5578, 5618, 5632, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44358 payload_rows_remote=38756 dup_rows=21178 dup_rows_remote=18514 dedup_ratio=0.323151 router_tokens_per_expert=[538, 523, 523, 503, 514, 505, 523, 504, 530, 535, 579, 531, 529, 537, 501, 488, 518, 520, 530, 518, 538, 480, 503, 510, 522, 512, 471, 490, 495, 488, 510, 480, 477, 523, 493, 490, 525, 506, 484, 534, 534, 544, 500, 533, 520, 481, 508, 503, 479, 494, 523, 483, 519, 478, 555, 511, 504, 507, 558, 519, 497, 520, 522, 496] payload_rows_per_dst=[5602, 5694, 5716, 5436, 5536, 5552, 5468, 5354] dup_rows_per_dst=[2664, 2766, 2978, 2500, 2528, 2694, 2616, 2432] combine_rows_per_src=[5602, 5694, 5716, 5436, 5536, 5552, 5468, 5354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44342 payload_rows_remote=38768 dup_rows=21194 dup_rows_remote=18432 dedup_ratio=0.323395 router_tokens_per_expert=[514, 509, 502, 533, 517, 519, 531, 520, 513, 499, 520, 531, 516, 506, 523, 480, 484, 529, 511, 503, 528, 502, 527, 506, 507, 504, 518, 532, 524, 489, 522, 535, 535, 471, 475, 511, 488, 491, 508, 492, 503, 501, 529, 518, 506, 509, 510, 525, 554, 534, 502, 498, 508, 543, 497, 532, 514, 528, 479, 519, 508, 502, 481, 543] payload_rows_per_dst=[5606, 5516, 5514, 5572, 5460, 5582, 5574, 5518] dup_rows_per_dst=[2684, 2660, 2666, 2690, 2482, 2620, 2762, 2630] combine_rows_per_src=[5606, 5516, 5514, 5572, 5460, 5582, 5574, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=160055296 dispatch_meta_bytes=229784 combine_payload_bytes=160055296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160285080 ingress_bytes=185221120 link_bytes=185221120 token_bytes=320340376 weight_movement_bytes=25165824 total_nvlink_bytes=345506200 weight_to_token_ratio=0.078560 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409318712e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=38820 dup_rows=21096 dup_rows_remote=18422 dedup_ratio=0.321899 router_tokens_per_expert=[531, 504, 522, 488, 500, 549, 499, 520, 498, 492, 507, 503, 542, 499, 514, 508, 521, 546, 561, 512, 481, 521, 479, 526, 475, 494, 491, 456, 482, 495, 531, 510, 510, 530, 493, 518, 481, 496, 476, 548, 548, 537, 523, 522, 517, 518, 526, 527, 547, 516, 534, 512, 519, 503, 488, 546, 491, 538, 521, 502, 503, 520, 494, 507] payload_rows_per_dst=[5526, 5540, 5620, 5516, 5506, 5654, 5556, 5522] dup_rows_per_dst=[2700, 2586, 2674, 2352, 2598, 2782, 2774, 2630] combine_rows_per_src=[5526, 5540, 5620, 5516, 5506, 5654, 5556, 5522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44494 payload_rows_remote=38976 dup_rows=21042 dup_rows_remote=18446 dedup_ratio=0.321075 router_tokens_per_expert=[477, 513, 505, 557, 500, 504, 508, 523, 525, 505, 495, 507, 500, 539, 475, 499, 523, 508, 551, 486, 524, 541, 529, 511, 528, 537, 485, 497, 490, 521, 490, 509, 521, 496, 540, 507, 521, 510, 525, 489, 500, 556, 498, 508, 509, 550, 484, 485, 508, 534, 517, 493, 489, 511, 487, 525, 503, 572, 488, 519, 507, 527, 519, 508] payload_rows_per_dst=[5594, 5520, 5592, 5518, 5556, 5588, 5526, 5600] dup_rows_per_dst=[2580, 2570, 2754, 2596, 2662, 2592, 2602, 2686] combine_rows_per_src=[5594, 5520, 5592, 5518, 5556, 5588, 5526, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159449088 dispatch_meta_bytes=229120 combine_payload_bytes=159449088 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159678208 ingress_bytes=209780736 link_bytes=209780736 token_bytes=319127296 weight_movement_bytes=50331648 total_nvlink_bytes=369458944 weight_to_token_ratio=0.157717 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596189013e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159989760 dispatch_meta_bytes=228648 combine_payload_bytes=159989760 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160218408 ingress_bytes=185155584 link_bytes=185155584 token_bytes=320208168 weight_movement_bytes=25165824 total_nvlink_bytes=345373992 weight_to_token_ratio=0.078592 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408820059e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158793728 dispatch_meta_bytes=228800 combine_payload_bytes=158793728 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159022528 ingress_bytes=183959552 link_bytes=183959552 token_bytes=317816256 weight_movement_bytes=25165824 total_nvlink_bytes=342982080 weight_to_token_ratio=0.079184 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399719637e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=158744576 dispatch_meta_bytes=229080 combine_payload_bytes=158744576 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=158973656 ingress_bytes=234242048 link_bytes=234242048 token_bytes=317718232 weight_movement_bytes=75497472 total_nvlink_bytes=393215704 weight_to_token_ratio=0.237624 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782311334e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44578 payload_rows_remote=39076 dup_rows=20958 dup_rows_remote=18276 dedup_ratio=0.319794 router_tokens_per_expert=[503, 518, 456, 487, 518, 516, 509, 519, 572, 525, 528, 532, 478, 502, 512, 498, 489, 489, 476, 501, 528, 506, 530, 537, 523, 511, 532, 501, 542, 538, 497, 478, 521, 520, 501, 484, 511, 542, 552, 533, 516, 539, 484, 513, 510, 533, 519, 478, 489, 510, 509, 549, 522, 519, 477, 501, 508, 570, 528, 460, 500, 506, 507, 506] payload_rows_per_dst=[5528, 5630, 5584, 5588, 5530, 5502, 5648, 5568] dup_rows_per_dst=[2524, 2664, 2528, 2656, 2798, 2682, 2504, 2602] combine_rows_per_src=[5528, 5630, 5584, 5588, 5530, 5502, 5648, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=159006720 dispatch_meta_bytes=228968 combine_payload_bytes=159006720 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159235688 ingress_bytes=259670016 link_bytes=259670016 token_bytes=318242408 weight_movement_bytes=100663296 total_nvlink_bytes=418905704 weight_to_token_ratio=0.316310 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.975788790e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159645696 dispatch_meta_bytes=229688 combine_payload_bytes=159645696 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159875384 ingress_bytes=260308992 link_bytes=260308992 token_bytes=319521080 weight_movement_bytes=100663296 total_nvlink_bytes=420184376 weight_to_token_ratio=0.315044 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.980650659e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44496 payload_rows_remote=38964 dup_rows=21040 dup_rows_remote=18394 dedup_ratio=0.321045 router_tokens_per_expert=[521, 523, 522, 500, 514, 509, 501, 522, 534, 525, 500, 510, 542, 508, 469, 501, 512, 538, 487, 461, 549, 481, 532, 516, 525, 509, 495, 547, 519, 497, 548, 493, 516, 495, 515, 506, 524, 475, 494, 519, 495, 542, 494, 520, 502, 501, 487, 516, 521, 513, 512, 506, 481, 518, 542, 531, 494, 495, 531, 495, 508, 533, 517, 560] payload_rows_per_dst=[5534, 5532, 5574, 5592, 5578, 5522, 5602, 5562] dup_rows_per_dst=[2690, 2646, 2578, 2674, 2510, 2592, 2646, 2704] combine_rows_per_src=[5534, 5532, 5574, 5592, 5578, 5522, 5602, 5562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160055296 dispatch_meta_bytes=229408 combine_payload_bytes=160055296 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160284704 ingress_bytes=185221120 link_bytes=185221120 token_bytes=320340000 weight_movement_bytes=25165824 total_nvlink_bytes=345505824 weight_to_token_ratio=0.078560 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.409318712e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44660 payload_rows_remote=39100 dup_rows=20876 dup_rows_remote=18248 dedup_ratio=0.318542 router_tokens_per_expert=[508, 518, 496, 519, 511, 501, 484, 525, 499, 512, 483, 549, 526, 582, 536, 480, 518, 504, 534, 504, 546, 532, 511, 520, 494, 505, 494, 493, 515, 503, 509, 508, 464, 544, 539, 516, 510, 544, 475, 536, 469, 574, 524, 508, 481, 479, 530, 503, 505, 495, 468, 516, 517, 508, 557, 495, 484, 497, 492, 506, 532, 571, 514, 496] payload_rows_per_dst=[5464, 5668, 5672, 5498, 5600, 5672, 5526, 5560] dup_rows_per_dst=[2426, 2666, 2666, 2520, 2656, 2698, 2616, 2628] combine_rows_per_src=[5464, 5668, 5672, 5498, 5600, 5672, 5526, 5560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=38808 dup_rows=21070 dup_rows_remote=18310 dedup_ratio=0.321503 router_tokens_per_expert=[508, 526, 500, 503, 509, 506, 551, 498, 491, 508, 522, 496, 475, 504, 551, 508, 498, 471, 523, 488, 512, 543, 544, 520, 514, 492, 542, 504, 527, 513, 469, 493, 528, 508, 498, 511, 531, 525, 510, 524, 516, 511, 522, 506, 541, 477, 500, 502, 506, 493, 473, 541, 540, 528, 544, 495, 519, 509, 525, 496, 502, 516, 521, 541] payload_rows_per_dst=[5658, 5564, 5520, 5552, 5594, 5448, 5628, 5502] dup_rows_per_dst=[2760, 2812, 2678, 2556, 2676, 2486, 2612, 2490] combine_rows_per_src=[5658, 5564, 5520, 5552, 5594, 5448, 5628, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44528 payload_rows_remote=38954 dup_rows=21008 dup_rows_remote=18450 dedup_ratio=0.320557 router_tokens_per_expert=[501, 567, 499, 519, 516, 482, 546, 512, 493, 519, 489, 515, 482, 504, 525, 501, 531, 518, 510, 513, 509, 553, 507, 512, 497, 515, 481, 506, 558, 501, 521, 494, 536, 495, 479, 515, 539, 503, 493, 506, 512, 497, 517, 514, 494, 480, 509, 510, 516, 554, 516, 518, 510, 542, 518, 513, 486, 503, 515, 564, 513, 494, 508, 503] payload_rows_per_dst=[5608, 5542, 5644, 5524, 5574, 5478, 5662, 5496] dup_rows_per_dst=[2676, 2514, 2662, 2622, 2558, 2588, 2712, 2676] combine_rows_per_src=[5608, 5542, 5644, 5524, 5574, 5478, 5662, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44572 payload_rows_remote=38938 dup_rows=20964 dup_rows_remote=18324 dedup_ratio=0.319885 router_tokens_per_expert=[453, 514, 522, 524, 524, 502, 509, 496, 534, 494, 504, 478, 510, 548, 521, 520, 539, 481, 480, 496, 526, 459, 494, 488, 518, 538, 521, 546, 531, 547, 505, 527, 487, 491, 483, 512, 514, 493, 513, 495, 500, 522, 528, 546, 524, 499, 533, 516, 517, 526, 502, 491, 545, 520, 530, 506, 509, 556, 500, 511, 535, 481, 491, 543] payload_rows_per_dst=[5540, 5578, 5458, 5692, 5530, 5604, 5634, 5536] dup_rows_per_dst=[2548, 2640, 2468, 2774, 2446, 2732, 2640, 2716] combine_rows_per_src=[5540, 5578, 5458, 5692, 5530, 5604, 5634, 5536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159596544 dispatch_meta_bytes=229432 combine_payload_bytes=159596544 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159825976 ingress_bytes=184762368 link_bytes=184762368 token_bytes=319422520 weight_movement_bytes=25165824 total_nvlink_bytes=344588344 weight_to_token_ratio=0.078785 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405828139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44402 payload_rows_remote=38772 dup_rows=21134 dup_rows_remote=18542 dedup_ratio=0.322479 router_tokens_per_expert=[546, 500, 512, 551, 536, 510, 540, 563, 521, 491, 528, 510, 525, 460, 521, 527, 528, 524, 511, 508, 521, 510, 513, 496, 525, 507, 485, 534, 504, 485, 501, 507, 509, 511, 522, 543, 506, 509, 500, 499, 494, 530, 516, 544, 518, 484, 468, 533, 531, 521, 518, 491, 545, 478, 504, 477, 501, 506, 513, 469, 515, 518, 517, 478] payload_rows_per_dst=[5708, 5502, 5630, 5526, 5532, 5520, 5514, 5470] dup_rows_per_dst=[2808, 2664, 2592, 2570, 2666, 2654, 2616, 2564] combine_rows_per_src=[5708, 5502, 5630, 5526, 5532, 5520, 5514, 5470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44690 payload_rows_remote=39262 dup_rows=20846 dup_rows_remote=18354 dedup_ratio=0.318085 router_tokens_per_expert=[545, 509, 481, 539, 507, 517, 505, 522, 506, 523, 542, 563, 546, 496, 526, 509, 517, 538, 504, 524, 508, 542, 537, 498, 487, 489, 483, 501, 516, 498, 500, 486, 485, 523, 501, 518, 482, 482, 523, 497, 491, 487, 511, 499, 506, 530, 509, 503, 511, 513, 536, 538, 537, 504, 530, 503, 470, 529, 513, 520, 534, 537, 482, 500] payload_rows_per_dst=[5600, 5630, 5612, 5428, 5582, 5558, 5702, 5578] dup_rows_per_dst=[2650, 2792, 2724, 2492, 2440, 2514, 2642, 2592] combine_rows_per_src=[5600, 5630, 5612, 5428, 5582, 5558, 5702, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=160153600 dispatch_meta_bytes=229392 combine_payload_bytes=160153600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160382992 ingress_bytes=185319424 link_bytes=185319424 token_bytes=320536592 weight_movement_bytes=25165824 total_nvlink_bytes=345702416 weight_to_token_ratio=0.078512 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410066692e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159555584 dispatch_meta_bytes=229616 combine_payload_bytes=159555584 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159785200 ingress_bytes=184721408 link_bytes=184721408 token_bytes=319340784 weight_movement_bytes=25165824 total_nvlink_bytes=344506608 weight_to_token_ratio=0.078806 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405516481e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44346 payload_rows_remote=38938 dup_rows=21190 dup_rows_remote=18614 dedup_ratio=0.323334 router_tokens_per_expert=[507, 528, 518, 498, 512, 480, 518, 527, 518, 533, 532, 499, 527, 507, 511, 513, 494, 480, 516, 549, 537, 501, 539, 502, 497, 489, 500, 531, 484, 520, 512, 540, 532, 536, 528, 517, 522, 480, 547, 519, 535, 509, 499, 503, 504, 501, 475, 466, 542, 523, 489, 500, 530, 497, 497, 533, 499, 534, 522, 492, 518, 493, 505, 502] payload_rows_per_dst=[5556, 5594, 5570, 5544, 5630, 5408, 5546, 5498] dup_rows_per_dst=[2620, 2686, 2666, 2602, 2732, 2576, 2676, 2632] combine_rows_per_src=[5556, 5594, 5570, 5544, 5630, 5408, 5546, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=158957568 dispatch_meta_bytes=228472 combine_payload_bytes=158957568 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159186040 ingress_bytes=209289216 link_bytes=209289216 token_bytes=318143608 weight_movement_bytes=50331648 total_nvlink_bytes=368475256 weight_to_token_ratio=0.158204 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.592449113e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159490048 dispatch_meta_bytes=229048 combine_payload_bytes=159490048 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159719096 ingress_bytes=234987520 link_bytes=234987520 token_bytes=319209144 weight_movement_bytes=75497472 total_nvlink_bytes=394706616 weight_to_token_ratio=0.236514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787983514e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=158810112 dispatch_meta_bytes=229256 combine_payload_bytes=158810112 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159039368 ingress_bytes=183975936 link_bytes=183975936 token_bytes=317849480 weight_movement_bytes=25165824 total_nvlink_bytes=343015304 weight_to_token_ratio=0.079175 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.399844301e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160817152 dispatch_meta_bytes=230464 combine_payload_bytes=160817152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=161047616 ingress_bytes=185982976 link_bytes=185982976 token_bytes=321864768 weight_movement_bytes=25165824 total_nvlink_bytes=347030592 weight_to_token_ratio=0.078188 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.415115556e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159490048 dispatch_meta_bytes=230208 combine_payload_bytes=159490048 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159720256 ingress_bytes=184655872 link_bytes=184655872 token_bytes=319210304 weight_movement_bytes=25165824 total_nvlink_bytes=344376128 weight_to_token_ratio=0.078838 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405017828e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44502 payload_rows_remote=38924 dup_rows=21034 dup_rows_remote=18388 dedup_ratio=0.320953 router_tokens_per_expert=[518, 505, 509, 508, 502, 513, 505, 519, 523, 545, 528, 481, 516, 497, 515, 507, 526, 503, 466, 515, 536, 502, 514, 535, 483, 518, 495, 538, 508, 491, 471, 499, 492, 488, 495, 528, 532, 529, 509, 477, 515, 498, 534, 529, 544, 509, 546, 532, 521, 506, 521, 510, 533, 529, 483, 507, 531, 526, 540, 501, 475, 500, 502, 535] payload_rows_per_dst=[5556, 5578, 5604, 5442, 5532, 5608, 5524, 5658] dup_rows_per_dst=[2602, 2646, 2590, 2564, 2568, 2806, 2696, 2562] combine_rows_per_src=[5556, 5578, 5604, 5442, 5532, 5608, 5524, 5658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44360 payload_rows_remote=38794 dup_rows=21176 dup_rows_remote=18584 dedup_ratio=0.323120 router_tokens_per_expert=[501, 534, 532, 526, 528, 534, 532, 508, 515, 495, 536, 496, 517, 535, 510, 521, 489, 468, 508, 508, 488, 532, 512, 481, 507, 543, 520, 461, 514, 552, 533, 518, 508, 538, 523, 542, 501, 525, 494, 504, 496, 530, 518, 507, 496, 477, 471, 496, 506, 512, 522, 507, 494, 552, 513, 535, 544, 499, 520, 513, 481, 510, 464, 516] payload_rows_per_dst=[5554, 5362, 5450, 5768, 5580, 5478, 5602, 5566] dup_rows_per_dst=[2680, 2366, 2558, 3034, 2746, 2504, 2696, 2592] combine_rows_per_src=[5554, 5362, 5450, 5768, 5580, 5478, 5602, 5566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44454 payload_rows_remote=38908 dup_rows=21082 dup_rows_remote=18438 dedup_ratio=0.321686 router_tokens_per_expert=[515, 517, 513, 523, 540, 504, 528, 517, 489, 505, 512, 534, 510, 505, 512, 520, 520, 497, 542, 531, 514, 532, 493, 542, 492, 538, 505, 491, 535, 487, 476, 538, 518, 534, 480, 546, 522, 522, 480, 493, 551, 579, 499, 498, 494, 499, 475, 539, 523, 478, 502, 470, 480, 530, 539, 489, 513, 483, 523, 539, 504, 493, 482, 514] payload_rows_per_dst=[5556, 5536, 5654, 5574, 5546, 5614, 5472, 5502] dup_rows_per_dst=[2758, 2638, 2688, 2550, 2644, 2654, 2550, 2600] combine_rows_per_src=[5556, 5536, 5654, 5574, 5546, 5614, 5472, 5502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44414 payload_rows_remote=38828 dup_rows=21122 dup_rows_remote=18400 dedup_ratio=0.322296 router_tokens_per_expert=[524, 497, 531, 499, 519, 533, 511, 511, 519, 495, 465, 494, 519, 519, 473, 489, 506, 496, 491, 540, 520, 499, 499, 486, 516, 540, 541, 561, 541, 491, 540, 520, 510, 538, 524, 498, 532, 510, 513, 481, 493, 514, 502, 489, 471, 533, 519, 535, 488, 561, 525, 492, 518, 503, 545, 505, 543, 498, 499, 505, 520, 527, 512, 480] payload_rows_per_dst=[5586, 5594, 5506, 5532, 5604, 5552, 5540, 5500] dup_rows_per_dst=[2722, 2606, 2628, 2654, 2736, 2560, 2548, 2668] combine_rows_per_src=[5586, 5594, 5506, 5532, 5604, 5552, 5540, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44478 payload_rows_remote=38906 dup_rows=21058 dup_rows_remote=18320 dedup_ratio=0.321320 router_tokens_per_expert=[488, 527, 506, 550, 508, 500, 507, 484, 492, 563, 489, 519, 526, 506, 520, 533, 498, 513, 517, 486, 520, 519, 499, 518, 526, 491, 518, 505, 514, 502, 514, 519, 560, 521, 490, 504, 486, 522, 486, 517, 526, 503, 481, 460, 493, 486, 510, 545, 511, 487, 535, 550, 521, 517, 515, 519, 555, 496, 546, 495, 513, 537, 519, 485] payload_rows_per_dst=[5534, 5582, 5526, 5582, 5568, 5516, 5572, 5598] dup_rows_per_dst=[2606, 2714, 2614, 2596, 2604, 2492, 2738, 2694] combine_rows_per_src=[5534, 5582, 5526, 5582, 5568, 5516, 5572, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=229248 combine_payload_bytes=159432704 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159661952 ingress_bytes=234930176 link_bytes=234930176 token_bytes=319094656 weight_movement_bytes=75497472 total_nvlink_bytes=394592128 weight_to_token_ratio=0.236599 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.787547193e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44480 payload_rows_remote=38874 dup_rows=21056 dup_rows_remote=18406 dedup_ratio=0.321289 router_tokens_per_expert=[527, 492, 539, 570, 486, 535, 514, 520, 500, 525, 537, 508, 505, 515, 542, 483, 549, 507, 505, 511, 517, 552, 521, 514, 514, 504, 521, 526, 487, 530, 523, 523, 482, 529, 527, 517, 520, 495, 520, 526, 518, 500, 504, 476, 511, 517, 532, 452, 537, 530, 490, 503, 522, 468, 471, 496, 492, 501, 507, 514, 490, 511, 517, 491] payload_rows_per_dst=[5588, 5654, 5632, 5606, 5608, 5468, 5508, 5416] dup_rows_per_dst=[2778, 2576, 2720, 2650, 2624, 2552, 2526, 2630] combine_rows_per_src=[5588, 5654, 5632, 5606, 5608, 5468, 5508, 5416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38846 dup_rows=21120 dup_rows_remote=18594 dedup_ratio=0.322266 router_tokens_per_expert=[498, 528, 514, 532, 510, 505, 505, 504, 523, 485, 511, 529, 504, 506, 519, 521, 487, 537, 501, 521, 515, 464, 520, 503, 514, 503, 518, 500, 547, 469, 513, 496, 499, 528, 529, 501, 455, 504, 524, 532, 493, 518, 546, 509, 526, 501, 531, 524, 496, 489, 506, 504, 557, 545, 522, 543, 511, 504, 495, 521, 531, 527, 498, 497] payload_rows_per_dst=[5546, 5462, 5570, 5546, 5526, 5574, 5640, 5552] dup_rows_per_dst=[2646, 2734, 2526, 2574, 2618, 2722, 2684, 2616] combine_rows_per_src=[5546, 5462, 5570, 5546, 5526, 5574, 5640, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=158900224 dispatch_meta_bytes=229512 combine_payload_bytes=158900224 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159129736 ingress_bytes=184066048 link_bytes=184066048 token_bytes=318029960 weight_movement_bytes=25165824 total_nvlink_bytes=343195784 weight_to_token_ratio=0.079130 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400529949e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44512 payload_rows_remote=38970 dup_rows=21024 dup_rows_remote=18374 dedup_ratio=0.320801 router_tokens_per_expert=[515, 534, 509, 500, 547, 501, 509, 502, 518, 521, 496, 510, 538, 504, 542, 502, 557, 489, 537, 474, 480, 519, 521, 523, 479, 492, 519, 488, 527, 481, 498, 530, 495, 509, 488, 515, 535, 514, 508, 527, 500, 497, 524, 549, 505, 503, 518, 500, 531, 521, 545, 529, 485, 530, 496, 497, 539, 507, 531, 523, 511, 489, 503, 482] payload_rows_per_dst=[5614, 5594, 5548, 5492, 5618, 5542, 5564, 5540] dup_rows_per_dst=[2620, 2668, 2652, 2536, 2564, 2650, 2704, 2630] combine_rows_per_src=[5614, 5594, 5548, 5492, 5618, 5542, 5564, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159367168 dispatch_meta_bytes=229384 combine_payload_bytes=159367168 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159596552 ingress_bytes=184532992 link_bytes=184532992 token_bytes=318963720 weight_movement_bytes=25165824 total_nvlink_bytes=344129544 weight_to_token_ratio=0.078899 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404082853e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=159039488 dispatch_meta_bytes=228912 combine_payload_bytes=159039488 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159268400 ingress_bytes=209371136 link_bytes=209371136 token_bytes=318307888 weight_movement_bytes=50331648 total_nvlink_bytes=368639536 weight_to_token_ratio=0.158123 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593072430e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=228904 combine_payload_bytes=159358976 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159587880 ingress_bytes=209690624 link_bytes=209690624 token_bytes=318946856 weight_movement_bytes=50331648 total_nvlink_bytes=369278504 weight_to_token_ratio=0.157806 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595503365e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159227904 dispatch_meta_bytes=229120 combine_payload_bytes=159227904 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159457024 ingress_bytes=209559552 link_bytes=209559552 token_bytes=318684928 weight_movement_bytes=50331648 total_nvlink_bytes=369016576 weight_to_token_ratio=0.157935 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.594506058e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=159113216 dispatch_meta_bytes=229760 combine_payload_bytes=159113216 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159342976 ingress_bytes=184279040 link_bytes=184279040 token_bytes=318456192 weight_movement_bytes=25165824 total_nvlink_bytes=343622016 weight_to_token_ratio=0.079024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402150572e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159621120 dispatch_meta_bytes=229376 combine_payload_bytes=159621120 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159850496 ingress_bytes=184786944 link_bytes=184786944 token_bytes=319471616 weight_movement_bytes=25165824 total_nvlink_bytes=344637440 weight_to_token_ratio=0.078773 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406015134e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44690 payload_rows_remote=39088 dup_rows=20846 dup_rows_remote=18186 dedup_ratio=0.318085 router_tokens_per_expert=[491, 563, 475, 515, 528, 498, 514, 472, 547, 511, 526, 513, 491, 527, 504, 512, 506, 525, 500, 506, 518, 513, 532, 571, 479, 505, 526, 497, 508, 486, 515, 535, 533, 504, 510, 493, 522, 514, 546, 499, 525, 502, 517, 514, 523, 494, 522, 500, 506, 518, 508, 489, 505, 491, 507, 531, 516, 539, 476, 504, 477, 539, 511, 524] payload_rows_per_dst=[5536, 5602, 5622, 5520, 5642, 5564, 5562, 5642] dup_rows_per_dst=[2576, 2660, 2720, 2582, 2600, 2630, 2548, 2530] combine_rows_per_src=[5536, 5602, 5622, 5520, 5642, 5564, 5562, 5642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44278 payload_rows_remote=38690 dup_rows=21258 dup_rows_remote=18522 dedup_ratio=0.324371 router_tokens_per_expert=[483, 550, 519, 483, 530, 485, 509, 529, 482, 530, 539, 525, 460, 533, 554, 501, 469, 496, 530, 533, 546, 490, 514, 530, 512, 510, 490, 532, 541, 488, 509, 506, 484, 527, 560, 497, 472, 536, 523, 476, 488, 484, 525, 507, 498, 530, 523, 524, 521, 522, 539, 497, 516, 471, 525, 532, 487, 505, 458, 519, 512, 511, 554, 537] payload_rows_per_dst=[5522, 5468, 5452, 5396, 5554, 5518, 5780, 5588] dup_rows_per_dst=[2610, 2622, 2508, 2488, 2596, 2640, 3058, 2736] combine_rows_per_src=[5522, 5468, 5452, 5396, 5554, 5518, 5780, 5588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44458 payload_rows_remote=38840 dup_rows=21078 dup_rows_remote=18456 dedup_ratio=0.321625 router_tokens_per_expert=[491, 459, 494, 474, 480, 514, 542, 502, 489, 505, 498, 497, 546, 505, 500, 511, 504, 516, 525, 481, 517, 503, 544, 544, 539, 496, 505, 537, 528, 500, 535, 546, 526, 501, 494, 497, 518, 519, 553, 512, 515, 495, 476, 543, 526, 496, 524, 516, 531, 538, 525, 509, 503, 514, 538, 490, 472, 460, 561, 518, 493, 503, 543, 532] payload_rows_per_dst=[5458, 5548, 5578, 5596, 5618, 5534, 5606, 5520] dup_rows_per_dst=[2454, 2554, 2690, 2776, 2622, 2648, 2690, 2644] combine_rows_per_src=[5458, 5548, 5578, 5596, 5618, 5534, 5606, 5520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44568 payload_rows_remote=39098 dup_rows=20968 dup_rows_remote=18576 dedup_ratio=0.319946 router_tokens_per_expert=[511, 487, 560, 514, 533, 532, 526, 498, 549, 510, 530, 514, 553, 514, 468, 496, 533, 475, 525, 519, 517, 555, 475, 503, 506, 536, 493, 506, 520, 497, 546, 518, 499, 503, 508, 481, 503, 520, 532, 530, 531, 519, 523, 524, 518, 467, 531, 519, 476, 476, 474, 508, 461, 504, 531, 501, 557, 467, 511, 521, 496, 520, 516, 522] payload_rows_per_dst=[5564, 5608, 5556, 5638, 5604, 5526, 5470, 5602] dup_rows_per_dst=[2758, 2660, 2648, 2606, 2548, 2738, 2392, 2618] combine_rows_per_src=[5564, 5608, 5556, 5638, 5604, 5526, 5470, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160104448 dispatch_meta_bytes=229096 combine_payload_bytes=160104448 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160333544 ingress_bytes=210436096 link_bytes=210436096 token_bytes=320437992 weight_movement_bytes=50331648 total_nvlink_bytes=370769640 weight_to_token_ratio=0.157071 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.601175545e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44606 payload_rows_remote=39088 dup_rows=20930 dup_rows_remote=18376 dedup_ratio=0.319366 router_tokens_per_expert=[519, 497, 490, 502, 530, 522, 491, 480, 506, 508, 553, 514, 552, 506, 502, 472, 520, 511, 520, 513, 509, 522, 508, 489, 522, 517, 503, 498, 491, 513, 541, 535, 530, 518, 542, 510, 544, 472, 486, 508, 529, 494, 526, 501, 503, 527, 523, 529, 510, 488, 505, 514, 491, 514, 485, 515, 508, 493, 505, 535, 534, 537, 499, 537] payload_rows_per_dst=[5518, 5508, 5588, 5564, 5640, 5740, 5396, 5652] dup_rows_per_dst=[2554, 2708, 2596, 2676, 2580, 2772, 2400, 2644] combine_rows_per_src=[5518, 5508, 5588, 5564, 5640, 5740, 5396, 5652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44504 payload_rows_remote=38894 dup_rows=21032 dup_rows_remote=18336 dedup_ratio=0.320923 router_tokens_per_expert=[516, 495, 515, 499, 506, 517, 533, 545, 508, 553, 486, 535, 502, 479, 486, 517, 498, 499, 519, 540, 498, 490, 513, 508, 488, 523, 539, 489, 536, 503, 541, 534, 484, 519, 500, 528, 552, 514, 499, 470, 498, 502, 545, 535, 503, 519, 530, 523, 495, 494, 513, 542, 513, 527, 518, 480, 481, 494, 506, 506, 533, 531, 510, 494] payload_rows_per_dst=[5624, 5486, 5550, 5610, 5526, 5634, 5536, 5538] dup_rows_per_dst=[2628, 2646, 2580, 2696, 2606, 2676, 2628, 2572] combine_rows_per_src=[5624, 5486, 5550, 5610, 5526, 5634, 5536, 5538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=158474240 dispatch_meta_bytes=228848 combine_payload_bytes=158474240 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158703088 ingress_bytes=183640064 link_bytes=183640064 token_bytes=317177328 weight_movement_bytes=25165824 total_nvlink_bytes=342343152 weight_to_token_ratio=0.079343 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.397288703e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44596 payload_rows_remote=39132 dup_rows=20940 dup_rows_remote=18460 dedup_ratio=0.319519 router_tokens_per_expert=[497, 530, 539, 506, 524, 491, 522, 515, 489, 498, 538, 495, 485, 513, 505, 505, 516, 495, 478, 501, 491, 506, 485, 500, 526, 503, 571, 502, 516, 570, 514, 532, 492, 507, 514, 461, 536, 533, 515, 505, 549, 494, 512, 524, 552, 484, 499, 504, 527, 501, 473, 512, 509, 502, 539, 516, 522, 508, 496, 549, 503, 553, 495, 524] payload_rows_per_dst=[5660, 5434, 5464, 5616, 5594, 5640, 5616, 5572] dup_rows_per_dst=[2588, 2622, 2480, 2852, 2532, 2596, 2542, 2728] combine_rows_per_src=[5660, 5434, 5464, 5616, 5594, 5640, 5616, 5572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44404 payload_rows_remote=38930 dup_rows=21132 dup_rows_remote=18592 dedup_ratio=0.322449 router_tokens_per_expert=[545, 534, 491, 526, 545, 541, 506, 505, 524, 529, 521, 523, 486, 500, 531, 510, 534, 547, 490, 503, 529, 522, 489, 463, 499, 524, 539, 490, 533, 485, 508, 539, 531, 537, 528, 558, 489, 505, 531, 465, 542, 496, 492, 504, 473, 479, 492, 529, 495, 501, 516, 501, 529, 484, 514, 539, 523, 495, 530, 500, 516, 478, 512, 473] payload_rows_per_dst=[5626, 5572, 5540, 5546, 5630, 5474, 5498, 5518] dup_rows_per_dst=[2760, 2676, 2614, 2688, 2658, 2540, 2660, 2536] combine_rows_per_src=[5626, 5572, 5540, 5546, 5630, 5474, 5498, 5518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=229184 combine_payload_bytes=159088640 prefetch_bytes=0 reduce_bytes=25165824 egress_bytes=159317824 ingress_bytes=184254464 link_bytes=184254464 token_bytes=318406464 weight_movement_bytes=25165824 total_nvlink_bytes=343572288 weight_to_token_ratio=0.079037 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401963577e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160145408 dispatch_meta_bytes=230696 combine_payload_bytes=160145408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160376104 ingress_bytes=185311232 link_bytes=185311232 token_bytes=320521512 weight_movement_bytes=25165824 total_nvlink_bytes=345687336 weight_to_token_ratio=0.078515 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410004360e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=160104448 dispatch_meta_bytes=229856 combine_payload_bytes=160104448 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=160334304 ingress_bytes=260767744 link_bytes=260767744 token_bytes=320438752 weight_movement_bytes=100663296 total_nvlink_bytes=421102048 weight_to_token_ratio=0.314142 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.984141231e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=228920 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159538744 ingress_bytes=184475648 link_bytes=184475648 token_bytes=318848568 weight_movement_bytes=25165824 total_nvlink_bytes=344014392 weight_to_token_ratio=0.078927 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403646531e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=160284672 dispatch_meta_bytes=230368 combine_payload_bytes=160284672 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160515040 ingress_bytes=210616320 link_bytes=210616320 token_bytes=320799712 weight_movement_bytes=50331648 total_nvlink_bytes=371131360 weight_to_token_ratio=0.156894 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.602546841e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159457280 dispatch_meta_bytes=230088 combine_payload_bytes=159457280 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159687368 ingress_bytes=184623104 link_bytes=184623104 token_bytes=319144648 weight_movement_bytes=25165824 total_nvlink_bytes=344310472 weight_to_token_ratio=0.078854 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404768501e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=38856 dup_rows=21096 dup_rows_remote=18600 dedup_ratio=0.321899 router_tokens_per_expert=[506, 502, 513, 511, 519, 532, 500, 497, 504, 487, 517, 507, 529, 481, 494, 521, 498, 515, 510, 525, 513, 510, 490, 492, 488, 507, 518, 517, 528, 513, 508, 522, 514, 525, 571, 519, 512, 562, 477, 518, 513, 482, 539, 495, 509, 509, 532, 479, 497, 532, 543, 508, 515, 507, 521, 486, 526, 482, 538, 520, 518, 504, 521, 520] payload_rows_per_dst=[5604, 5584, 5506, 5518, 5632, 5462, 5578, 5556] dup_rows_per_dst=[2556, 2496, 2600, 2684, 2764, 2654, 2640, 2702] combine_rows_per_src=[5604, 5584, 5506, 5518, 5632, 5462, 5578, 5556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44396 payload_rows_remote=38886 dup_rows=21140 dup_rows_remote=18564 dedup_ratio=0.322571 router_tokens_per_expert=[504, 529, 570, 501, 526, 513, 507, 492, 506, 525, 518, 519, 490, 498, 490, 502, 486, 513, 524, 511, 534, 531, 523, 489, 494, 539, 501, 508, 486, 499, 512, 544, 467, 506, 536, 554, 483, 482, 514, 562, 492, 527, 471, 508, 485, 513, 526, 547, 524, 533, 571, 491, 501, 522, 529, 528, 486, 485, 490, 514, 528, 477, 510, 522] payload_rows_per_dst=[5466, 5596, 5346, 5532, 5554, 5608, 5784, 5510] dup_rows_per_dst=[2554, 2764, 2322, 2634, 2592, 2810, 2888, 2576] combine_rows_per_src=[5466, 5596, 5346, 5532, 5554, 5608, 5784, 5510] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44606 payload_rows_remote=39062 dup_rows=20930 dup_rows_remote=18314 dedup_ratio=0.319366 router_tokens_per_expert=[525, 451, 507, 525, 492, 511, 478, 531, 511, 517, 483, 527, 568, 481, 486, 512, 535, 513, 527, 504, 535, 523, 522, 512, 519, 490, 507, 515, 525, 479, 520, 525, 511, 518, 524, 525, 473, 540, 506, 483, 517, 505, 515, 490, 511, 524, 520, 516, 524, 514, 535, 504, 495, 537, 512, 530, 514, 522, 515, 521, 510, 490, 486, 525] payload_rows_per_dst=[5460, 5588, 5634, 5540, 5544, 5646, 5636, 5558] dup_rows_per_dst=[2580, 2582, 2708, 2620, 2616, 2550, 2666, 2608] combine_rows_per_src=[5460, 5588, 5634, 5540, 5544, 5646, 5636, 5558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44530 payload_rows_remote=39100 dup_rows=21006 dup_rows_remote=18390 dedup_ratio=0.320526 router_tokens_per_expert=[518, 509, 548, 525, 510, 518, 506, 504, 517, 509, 511, 506, 530, 531, 493, 499, 517, 514, 551, 545, 539, 507, 509, 540, 513, 536, 481, 469, 510, 501, 540, 503, 529, 535, 466, 558, 513, 519, 485, 480, 507, 479, 535, 481, 547, 475, 522, 474, 509, 509, 517, 504, 506, 473, 518, 487, 536, 570, 533, 497, 520, 481, 489, 505] payload_rows_per_dst=[5680, 5566, 5612, 5540, 5548, 5546, 5430, 5608] dup_rows_per_dst=[2596, 2626, 2832, 2566, 2622, 2494, 2616, 2654] combine_rows_per_src=[5680, 5566, 5612, 5540, 5548, 5546, 5430, 5608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=229824 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159384000 ingress_bytes=184320000 link_bytes=184320000 token_bytes=318538176 weight_movement_bytes=25165824 total_nvlink_bytes=343704000 weight_to_token_ratio=0.079004 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402462230e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44608 payload_rows_remote=39058 dup_rows=20928 dup_rows_remote=18268 dedup_ratio=0.319336 router_tokens_per_expert=[505, 547, 489, 501, 538, 510, 503, 508, 478, 527, 493, 535, 524, 496, 515, 547, 526, 534, 501, 530, 527, 486, 522, 507, 527, 480, 517, 487, 539, 520, 510, 511, 507, 539, 499, 501, 530, 512, 503, 533, 521, 543, 460, 542, 540, 511, 541, 491, 484, 504, 463, 488, 484, 522, 519, 521, 505, 511, 512, 511, 501, 495, 513, 522] payload_rows_per_dst=[5550, 5604, 5596, 5600, 5590, 5642, 5452, 5574] dup_rows_per_dst=[2660, 2662, 2670, 2582, 2622, 2648, 2518, 2566] combine_rows_per_src=[5550, 5604, 5596, 5600, 5590, 5642, 5452, 5574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44560 payload_rows_remote=39034 dup_rows=20976 dup_rows_remote=18390 dedup_ratio=0.320068 router_tokens_per_expert=[524, 496, 542, 498, 495, 497, 519, 480, 502, 507, 514, 503, 507, 515, 531, 512, 547, 514, 528, 499, 502, 486, 505, 490, 563, 525, 488, 491, 479, 499, 496, 515, 527, 530, 469, 549, 523, 501, 535, 539, 478, 502, 513, 506, 507, 506, 517, 526, 533, 518, 484, 519, 482, 492, 499, 522, 531, 514, 535, 550, 553, 517, 489, 533] payload_rows_per_dst=[5594, 5532, 5512, 5526, 5674, 5558, 5532, 5632] dup_rows_per_dst=[2508, 2650, 2630, 2586, 2672, 2552, 2566, 2812] combine_rows_per_src=[5594, 5532, 5512, 5526, 5674, 5558, 5532, 5632] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159277056 dispatch_meta_bytes=229800 combine_payload_bytes=159277056 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159506856 ingress_bytes=184442880 link_bytes=184442880 token_bytes=318783912 weight_movement_bytes=25165824 total_nvlink_bytes=343949736 weight_to_token_ratio=0.078943 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403397205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44378 payload_rows_remote=38838 dup_rows=21158 dup_rows_remote=18520 dedup_ratio=0.322845 router_tokens_per_expert=[518, 505, 519, 475, 548, 494, 543, 551, 525, 528, 463, 505, 523, 519, 540, 536, 498, 535, 509, 511, 505, 513, 554, 499, 509, 520, 507, 478, 497, 491, 482, 512, 507, 505, 505, 545, 483, 502, 517, 542, 484, 506, 501, 541, 491, 535, 532, 499, 521, 477, 503, 529, 512, 488, 496, 526, 510, 511, 496, 501, 503, 527, 528, 533] payload_rows_per_dst=[5598, 5550, 5508, 5470, 5614, 5540, 5520, 5578] dup_rows_per_dst=[2708, 2728, 2740, 2522, 2598, 2638, 2584, 2640] combine_rows_per_src=[5598, 5550, 5508, 5470, 5614, 5540, 5520, 5578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44422 payload_rows_remote=38806 dup_rows=21114 dup_rows_remote=18396 dedup_ratio=0.322174 router_tokens_per_expert=[529, 482, 534, 527, 483, 530, 513, 543, 484, 533, 524, 497, 526, 525, 483, 494, 534, 506, 521, 528, 498, 497, 567, 516, 472, 488, 498, 539, 522, 489, 510, 508, 516, 526, 526, 501, 479, 534, 481, 533, 532, 533, 509, 490, 503, 531, 490, 521, 500, 521, 474, 500, 513, 487, 510, 502, 504, 564, 523, 522, 530, 508, 530, 475] payload_rows_per_dst=[5628, 5522, 5616, 5504, 5590, 5556, 5420, 5586] dup_rows_per_dst=[2654, 2610, 2718, 2548, 2602, 2662, 2594, 2726] combine_rows_per_src=[5628, 5522, 5616, 5504, 5590, 5556, 5420, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159997952 dispatch_meta_bytes=229504 combine_payload_bytes=159997952 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=160227456 ingress_bytes=260661248 link_bytes=260661248 token_bytes=320225408 weight_movement_bytes=100663296 total_nvlink_bytes=420888704 weight_to_token_ratio=0.314351 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.983330920e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=160153600 dispatch_meta_bytes=229960 combine_payload_bytes=160153600 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160383560 ingress_bytes=185319424 link_bytes=185319424 token_bytes=320537160 weight_movement_bytes=25165824 total_nvlink_bytes=345702984 weight_to_token_ratio=0.078511 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.410066692e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=159981568 dispatch_meta_bytes=229304 combine_payload_bytes=159981568 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=160210872 ingress_bytes=210313216 link_bytes=210313216 token_bytes=320192440 weight_movement_bytes=50331648 total_nvlink_bytes=370524088 weight_to_token_ratio=0.157192 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.600240570e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=229696 combine_payload_bytes=159883264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160112960 ingress_bytes=185049088 link_bytes=185049088 token_bytes=319996224 weight_movement_bytes=25165824 total_nvlink_bytes=345162048 weight_to_token_ratio=0.078644 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408009747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=158949376 dispatch_meta_bytes=228808 combine_payload_bytes=158949376 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159178184 ingress_bytes=234446848 link_bytes=234446848 token_bytes=318127560 weight_movement_bytes=75497472 total_nvlink_bytes=393625032 weight_to_token_ratio=0.237318 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.783869625e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159080448 dispatch_meta_bytes=229432 combine_payload_bytes=159080448 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159309880 ingress_bytes=209412096 link_bytes=209412096 token_bytes=318390328 weight_movement_bytes=50331648 total_nvlink_bytes=368721976 weight_to_token_ratio=0.158082 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593384088e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44356 payload_rows_remote=38840 dup_rows=21180 dup_rows_remote=18486 dedup_ratio=0.323181 router_tokens_per_expert=[517, 517, 514, 505, 536, 500, 470, 469, 509, 527, 495, 505, 516, 499, 521, 533, 545, 518, 495, 544, 541, 498, 530, 549, 495, 501, 499, 514, 489, 532, 501, 562, 522, 503, 528, 511, 527, 512, 530, 475, 509, 521, 471, 516, 516, 521, 488, 492, 513, 492, 516, 491, 500, 524, 491, 534, 516, 505, 540, 459, 509, 540, 524, 526] payload_rows_per_dst=[5454, 5516, 5692, 5530, 5562, 5506, 5590, 5506] dup_rows_per_dst=[2602, 2694, 2748, 2656, 2654, 2562, 2532, 2732] combine_rows_per_src=[5454, 5516, 5692, 5530, 5562, 5506, 5590, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44438 payload_rows_remote=38898 dup_rows=21098 dup_rows_remote=18394 dedup_ratio=0.321930 router_tokens_per_expert=[493, 489, 509, 573, 506, 545, 486, 499, 495, 487, 498, 454, 533, 505, 518, 511, 508, 523, 548, 502, 530, 506, 507, 500, 512, 496, 506, 507, 493, 519, 486, 508, 511, 491, 525, 520, 464, 510, 521, 517, 500, 514, 499, 513, 532, 548, 528, 527, 524, 532, 508, 493, 521, 533, 535, 528, 515, 512, 523, 510, 546, 488, 521, 507] payload_rows_per_dst=[5616, 5448, 5264, 5782, 5556, 5628, 5604, 5540] dup_rows_per_dst=[2584, 2554, 2396, 2860, 2562, 2694, 2744, 2704] combine_rows_per_src=[5616, 5448, 5264, 5782, 5556, 5628, 5604, 5540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44336 payload_rows_remote=38866 dup_rows=21200 dup_rows_remote=18584 dedup_ratio=0.323486 router_tokens_per_expert=[543, 477, 552, 490, 470, 515, 466, 517, 499, 527, 537, 501, 505, 521, 495, 522, 490, 535, 495, 528, 503, 525, 526, 471, 506, 540, 507, 568, 502, 489, 499, 519, 507, 524, 500, 487, 498, 510, 486, 531, 514, 479, 507, 468, 550, 527, 556, 528, 509, 502, 532, 516, 508, 519, 491, 551, 508, 522, 501, 526, 514, 483, 549, 525] payload_rows_per_dst=[5474, 5606, 5518, 5590, 5470, 5578, 5510, 5590] dup_rows_per_dst=[2586, 2608, 2628, 2670, 2616, 2680, 2746, 2666] combine_rows_per_src=[5474, 5606, 5518, 5590, 5470, 5578, 5510, 5590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159088640 dispatch_meta_bytes=229304 combine_payload_bytes=159088640 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159317944 ingress_bytes=184254464 link_bytes=184254464 token_bytes=318406584 weight_movement_bytes=25165824 total_nvlink_bytes=343572408 weight_to_token_ratio=0.079037 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401963577e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44668 payload_rows_remote=39132 dup_rows=20868 dup_rows_remote=18366 dedup_ratio=0.318420 router_tokens_per_expert=[536, 497, 472, 507, 491, 525, 537, 524, 524, 504, 516, 506, 512, 511, 499, 496, 520, 500, 512, 531, 531, 497, 541, 520, 516, 518, 487, 441, 538, 506, 512, 508, 506, 509, 519, 550, 522, 508, 517, 476, 545, 541, 533, 519, 532, 510, 513, 511, 506, 501, 486, 487, 517, 493, 482, 547, 537, 502, 565, 501, 497, 481, 510, 510] payload_rows_per_dst=[5566, 5596, 5598, 5536, 5624, 5610, 5536, 5602] dup_rows_per_dst=[2612, 2540, 2706, 2516, 2590, 2798, 2502, 2604] combine_rows_per_src=[5566, 5596, 5598, 5536, 5624, 5610, 5536, 5602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44488 payload_rows_remote=38922 dup_rows=21048 dup_rows_remote=18494 dedup_ratio=0.321167 router_tokens_per_expert=[471, 495, 496, 519, 537, 501, 493, 530, 509, 518, 534, 519, 514, 480, 513, 524, 523, 514, 550, 547, 511, 495, 521, 513, 512, 509, 490, 553, 521, 500, 493, 523, 546, 535, 522, 511, 482, 494, 522, 491, 499, 485, 506, 555, 512, 492, 510, 553, 462, 519, 490, 468, 473, 515, 539, 515, 489, 509, 533, 527, 516, 516, 532, 522] payload_rows_per_dst=[5566, 5604, 5660, 5544, 5584, 5480, 5452, 5598] dup_rows_per_dst=[2554, 2618, 2698, 2766, 2468, 2744, 2510, 2690] combine_rows_per_src=[5566, 5604, 5660, 5544, 5584, 5480, 5452, 5598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44486 payload_rows_remote=38916 dup_rows=21050 dup_rows_remote=18450 dedup_ratio=0.321198 router_tokens_per_expert=[525, 514, 524, 549, 483, 509, 503, 466, 505, 493, 522, 513, 505, 513, 497, 480, 485, 518, 511, 511, 508, 500, 529, 547, 501, 520, 528, 508, 526, 492, 488, 522, 515, 530, 531, 480, 489, 523, 535, 535, 502, 517, 510, 517, 514, 501, 526, 494, 489, 522, 487, 513, 509, 547, 526, 481, 544, 546, 546, 520, 500, 522, 505, 497] payload_rows_per_dst=[5552, 5502, 5560, 5570, 5582, 5596, 5524, 5600] dup_rows_per_dst=[2594, 2554, 2658, 2600, 2694, 2566, 2624, 2760] combine_rows_per_src=[5552, 5502, 5560, 5570, 5582, 5596, 5524, 5600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159326208 dispatch_meta_bytes=229168 combine_payload_bytes=159326208 prefetch_bytes=0 reduce_bytes=100663296 egress_bytes=159555376 ingress_bytes=259989504 link_bytes=259989504 token_bytes=318881584 weight_movement_bytes=100663296 total_nvlink_bytes=419544880 weight_to_token_ratio=0.315676 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.978219724e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44528 payload_rows_remote=38940 dup_rows=21008 dup_rows_remote=18398 dedup_ratio=0.320557 router_tokens_per_expert=[530, 513, 498, 507, 531, 515, 476, 538, 514, 526, 513, 515, 546, 534, 514, 494, 532, 517, 495, 532, 490, 494, 511, 528, 558, 473, 511, 499, 517, 492, 480, 534, 515, 486, 498, 528, 477, 509, 531, 503, 512, 471, 476, 504, 509, 524, 508, 548, 528, 502, 494, 514, 547, 486, 511, 532, 493, 514, 514, 513, 513, 522, 532, 527] payload_rows_per_dst=[5558, 5616, 5588, 5552, 5582, 5482, 5604, 5546] dup_rows_per_dst=[2658, 2696, 2610, 2576, 2512, 2622, 2624, 2710] combine_rows_per_src=[5558, 5616, 5588, 5552, 5582, 5482, 5604, 5546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159195136 dispatch_meta_bytes=229800 combine_payload_bytes=159195136 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159424936 ingress_bytes=184360960 link_bytes=184360960 token_bytes=318620072 weight_movement_bytes=25165824 total_nvlink_bytes=343785896 weight_to_token_ratio=0.078984 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402773888e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44298 payload_rows_remote=38732 dup_rows=21238 dup_rows_remote=18476 dedup_ratio=0.324066 router_tokens_per_expert=[500, 485, 533, 539, 512, 529, 509, 534, 493, 493, 516, 515, 474, 515, 549, 490, 555, 517, 522, 510, 493, 514, 511, 518, 519, 472, 481, 557, 551, 544, 503, 544, 513, 497, 475, 520, 517, 495, 476, 494, 528, 523, 537, 554, 497, 487, 517, 521, 528, 553, 491, 535, 510, 497, 514, 493, 514, 530, 488, 480, 471, 526, 507, 483] payload_rows_per_dst=[5532, 5500, 5522, 5644, 5488, 5566, 5580, 5466] dup_rows_per_dst=[2750, 2590, 2758, 2698, 2486, 2762, 2662, 2532] combine_rows_per_src=[5532, 5500, 5522, 5644, 5488, 5566, 5580, 5466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=160284672 dispatch_meta_bytes=229992 combine_payload_bytes=160284672 prefetch_bytes=0 reduce_bytes=0 egress_bytes=160514664 ingress_bytes=160284672 link_bytes=160514664 token_bytes=320799336 weight_movement_bytes=0 total_nvlink_bytes=320799336 weight_to_token_ratio=0.000000 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.221331129e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=229664 combine_payload_bytes=159424512 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159654176 ingress_bytes=209756160 link_bytes=209756160 token_bytes=319078688 weight_movement_bytes=50331648 total_nvlink_bytes=369410336 weight_to_token_ratio=0.157741 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596002018e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159399936 dispatch_meta_bytes=229464 combine_payload_bytes=159399936 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159629400 ingress_bytes=184565760 link_bytes=184565760 token_bytes=319029336 weight_movement_bytes=25165824 total_nvlink_bytes=344195160 weight_to_token_ratio=0.078882 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404332180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159498240 dispatch_meta_bytes=229352 combine_payload_bytes=159498240 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159727592 ingress_bytes=209829888 link_bytes=209829888 token_bytes=319225832 weight_movement_bytes=50331648 total_nvlink_bytes=369557480 weight_to_token_ratio=0.157668 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596563003e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44514 payload_rows_remote=38912 dup_rows=21022 dup_rows_remote=18374 dedup_ratio=0.320770 router_tokens_per_expert=[519, 490, 497, 509, 531, 538, 533, 512, 497, 546, 519, 527, 498, 521, 497, 520, 535, 492, 491, 501, 501, 513, 498, 507, 504, 508, 480, 518, 503, 480, 499, 511, 509, 492, 517, 507, 530, 489, 522, 520, 531, 518, 486, 528, 489, 490, 543, 528, 506, 516, 531, 496, 533, 541, 542, 516, 480, 540, 497, 505, 534, 499, 483, 555] payload_rows_per_dst=[5606, 5602, 5546, 5526, 5566, 5538, 5634, 5496] dup_rows_per_dst=[2652, 2648, 2530, 2480, 2606, 2688, 2728, 2690] combine_rows_per_src=[5606, 5602, 5546, 5526, 5566, 5538, 5634, 5496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158646272 dispatch_meta_bytes=228832 combine_payload_bytes=158646272 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=158875104 ingress_bytes=208977920 link_bytes=208977920 token_bytes=317521376 weight_movement_bytes=50331648 total_nvlink_bytes=367853024 weight_to_token_ratio=0.158514 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.590080511e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44276 payload_rows_remote=38776 dup_rows=21260 dup_rows_remote=18494 dedup_ratio=0.324402 router_tokens_per_expert=[554, 510, 501, 509, 489, 513, 526, 557, 544, 549, 494, 516, 533, 528, 522, 488, 509, 503, 503, 524, 521, 527, 547, 546, 467, 477, 492, 518, 479, 493, 498, 499, 525, 488, 537, 543, 488, 534, 554, 522, 480, 536, 517, 492, 461, 489, 483, 515, 480, 528, 509, 534, 485, 525, 511, 480, 537, 544, 511, 528, 523, 508, 482, 483] payload_rows_per_dst=[5600, 5566, 5660, 5332, 5676, 5298, 5644, 5500] dup_rows_per_dst=[2718, 2748, 2700, 2514, 2706, 2254, 2854, 2766] combine_rows_per_src=[5600, 5566, 5660, 5332, 5676, 5298, 5644, 5500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159383552 dispatch_meta_bytes=229144 combine_payload_bytes=159383552 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159612696 ingress_bytes=209715200 link_bytes=209715200 token_bytes=318996248 weight_movement_bytes=50331648 total_nvlink_bytes=369327896 weight_to_token_ratio=0.157781 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.595690359e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44396 payload_rows_remote=38856 dup_rows=21140 dup_rows_remote=18336 dedup_ratio=0.322571 router_tokens_per_expert=[518, 483, 523, 506, 464, 489, 539, 503, 503, 511, 494, 532, 509, 475, 486, 581, 512, 517, 474, 526, 506, 509, 530, 511, 507, 504, 538, 511, 509, 460, 511, 534, 517, 515, 524, 509, 520, 570, 523, 494, 551, 514, 536, 539, 511, 541, 506, 533, 519, 515, 536, 475, 493, 510, 543, 467, 471, 545, 498, 524, 461, 511, 488, 534] payload_rows_per_dst=[5544, 5544, 5630, 5512, 5540, 5666, 5510, 5450] dup_rows_per_dst=[2506, 2638, 2540, 2636, 2804, 2796, 2606, 2614] combine_rows_per_src=[5544, 5544, 5630, 5512, 5540, 5666, 5510, 5450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44440 payload_rows_remote=38942 dup_rows=21096 dup_rows_remote=18602 dedup_ratio=0.321899 router_tokens_per_expert=[490, 498, 523, 484, 499, 486, 521, 488, 562, 480, 508, 480, 502, 503, 530, 541, 485, 485, 544, 553, 523, 486, 480, 540, 540, 478, 495, 477, 534, 527, 521, 501, 544, 500, 551, 539, 515, 515, 474, 469, 508, 532, 547, 539, 496, 585, 474, 515, 502, 484, 461, 484, 523, 505, 520, 517, 541, 523, 539, 533, 488, 531, 526, 524] payload_rows_per_dst=[5462, 5534, 5598, 5540, 5576, 5622, 5498, 5610] dup_rows_per_dst=[2516, 2678, 2594, 2606, 2638, 2770, 2494, 2800] combine_rows_per_src=[5462, 5534, 5598, 5540, 5576, 5622, 5498, 5610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44452 payload_rows_remote=38848 dup_rows=21084 dup_rows_remote=18270 dedup_ratio=0.321716 router_tokens_per_expert=[493, 529, 505, 495, 522, 528, 482, 501, 544, 488, 516, 525, 474, 522, 493, 518, 527, 530, 510, 486, 565, 547, 498, 510, 506, 519, 516, 510, 495, 530, 520, 486, 525, 557, 489, 481, 507, 526, 478, 467, 502, 511, 515, 517, 535, 546, 500, 512, 502, 499, 524, 536, 511, 508, 561, 530, 549, 538, 504, 484, 492, 488, 488, 496] payload_rows_per_dst=[5604, 5596, 5662, 5560, 5562, 5474, 5654, 5340] dup_rows_per_dst=[2814, 2606, 2736, 2604, 2688, 2614, 2688, 2334] combine_rows_per_src=[5604, 5596, 5662, 5560, 5562, 5474, 5654, 5340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44504 payload_rows_remote=38952 dup_rows=21032 dup_rows_remote=18304 dedup_ratio=0.320923 router_tokens_per_expert=[504, 486, 469, 509, 481, 484, 532, 525, 533, 539, 490, 468, 530, 516, 481, 559, 493, 508, 541, 487, 515, 572, 532, 480, 548, 526, 526, 501, 507, 524, 502, 506, 518, 487, 484, 463, 497, 530, 526, 530, 474, 517, 533, 526, 514, 496, 510, 511, 543, 495, 534, 504, 515, 498, 527, 526, 537, 510, 521, 550, 504, 504, 503, 507] payload_rows_per_dst=[5506, 5604, 5580, 5552, 5542, 5498, 5592, 5630] dup_rows_per_dst=[2474, 2628, 2676, 2728, 2528, 2664, 2692, 2642] combine_rows_per_src=[5506, 5604, 5580, 5552, 5542, 5498, 5592, 5630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=158826496 dispatch_meta_bytes=229080 combine_payload_bytes=158826496 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159055576 ingress_bytes=234323968 link_bytes=234323968 token_bytes=317882072 weight_movement_bytes=75497472 total_nvlink_bytes=393379544 weight_to_token_ratio=0.237502 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.782934650e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44740 payload_rows_remote=39148 dup_rows=20796 dup_rows_remote=18248 dedup_ratio=0.317322 router_tokens_per_expert=[482, 497, 506, 502, 530, 546, 485, 526, 531, 521, 499, 515, 537, 518, 507, 543, 500, 509, 497, 506, 525, 482, 503, 548, 510, 549, 487, 505, 522, 533, 478, 490, 515, 499, 484, 518, 537, 468, 531, 558, 526, 516, 507, 483, 509, 468, 561, 530, 481, 528, 546, 491, 527, 476, 496, 515, 549, 514, 486, 500, 525, 472, 529, 534] payload_rows_per_dst=[5560, 5658, 5592, 5602, 5552, 5626, 5582, 5568] dup_rows_per_dst=[2588, 2684, 2548, 2546, 2668, 2574, 2538, 2650] combine_rows_per_src=[5560, 5658, 5592, 5602, 5552, 5626, 5582, 5568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=228768 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159382944 ingress_bytes=209485824 link_bytes=209485824 token_bytes=318537120 weight_movement_bytes=50331648 total_nvlink_bytes=368868768 weight_to_token_ratio=0.158009 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.593945073e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159506432 dispatch_meta_bytes=230176 combine_payload_bytes=159506432 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159736608 ingress_bytes=184672256 link_bytes=184672256 token_bytes=319243040 weight_movement_bytes=25165824 total_nvlink_bytes=344408864 weight_to_token_ratio=0.078830 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.405142491e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38894 dup_rows=21120 dup_rows_remote=18558 dedup_ratio=0.322266 router_tokens_per_expert=[494, 489, 500, 525, 496, 510, 537, 479, 526, 510, 535, 516, 525, 525, 504, 534, 488, 515, 499, 528, 544, 492, 531, 541, 493, 484, 503, 526, 517, 519, 504, 500, 482, 465, 546, 489, 545, 515, 490, 517, 481, 541, 524, 490, 483, 495, 518, 510, 488, 532, 516, 510, 504, 529, 537, 510, 491, 487, 517, 564, 528, 519, 521, 535] payload_rows_per_dst=[5460, 5646, 5598, 5460, 5526, 5522, 5588, 5616] dup_rows_per_dst=[2600, 2704, 2678, 2632, 2572, 2562, 2664, 2708] combine_rows_per_src=[5460, 5646, 5598, 5460, 5526, 5522, 5588, 5616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=159121408 dispatch_meta_bytes=228472 combine_payload_bytes=159121408 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159349880 ingress_bytes=184287232 link_bytes=184287232 token_bytes=318471288 weight_movement_bytes=25165824 total_nvlink_bytes=343637112 weight_to_token_ratio=0.079021 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.402212903e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159547392 dispatch_meta_bytes=229024 combine_payload_bytes=159547392 prefetch_bytes=25165824 reduce_bytes=50331648 egress_bytes=159776416 ingress_bytes=235044864 link_bytes=235044864 token_bytes=319323808 weight_movement_bytes=75497472 total_nvlink_bytes=394821280 weight_to_token_ratio=0.236429 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.788419836e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44548 payload_rows_remote=38932 dup_rows=20988 dup_rows_remote=18262 dedup_ratio=0.320251 router_tokens_per_expert=[529, 481, 513, 481, 509, 525, 493, 547, 495, 489, 495, 544, 516, 484, 594, 554, 502, 493, 521, 479, 521, 525, 531, 530, 521, 479, 503, 536, 489, 483, 500, 492, 519, 503, 514, 532, 498, 528, 504, 504, 546, 530, 518, 497, 465, 539, 555, 498, 505, 482, 526, 487, 517, 517, 567, 506, 486, 518, 512, 486, 521, 551, 504, 479] payload_rows_per_dst=[5546, 5616, 5578, 5494, 5604, 5610, 5546, 5554] dup_rows_per_dst=[2610, 2726, 2626, 2512, 2600, 2686, 2668, 2560] combine_rows_per_src=[5546, 5616, 5578, 5494, 5604, 5610, 5546, 5554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=160350208 dispatch_meta_bytes=229584 combine_payload_bytes=160350208 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=160579792 ingress_bytes=210681856 link_bytes=210681856 token_bytes=320930000 weight_movement_bytes=50331648 total_nvlink_bytes=371261648 weight_to_token_ratio=0.156831 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.603045495e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159309824 dispatch_meta_bytes=229808 combine_payload_bytes=159309824 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159539632 ingress_bytes=184475648 link_bytes=184475648 token_bytes=318849456 weight_movement_bytes=25165824 total_nvlink_bytes=344015280 weight_to_token_ratio=0.078927 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403646531e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44524 payload_rows_remote=38856 dup_rows=21012 dup_rows_remote=18252 dedup_ratio=0.320618 router_tokens_per_expert=[510, 501, 508, 572, 511, 492, 481, 500, 492, 487, 510, 502, 503, 535, 549, 499, 515, 508, 500, 492, 494, 546, 517, 491, 499, 516, 497, 518, 535, 498, 494, 537, 512, 490, 542, 499, 513, 494, 483, 477, 499, 541, 524, 508, 544, 469, 524, 515, 529, 515, 503, 539, 540, 516, 506, 485, 526, 517, 516, 524, 545, 527, 510, 527] payload_rows_per_dst=[5572, 5602, 5548, 5502, 5504, 5534, 5594, 5668] dup_rows_per_dst=[2578, 2552, 2578, 2642, 2516, 2714, 2672, 2760] combine_rows_per_src=[5572, 5602, 5548, 5502, 5504, 5534, 5594, 5668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159465472 dispatch_meta_bytes=228776 combine_payload_bytes=159465472 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159694248 ingress_bytes=184631296 link_bytes=184631296 token_bytes=319159720 weight_movement_bytes=25165824 total_nvlink_bytes=344325544 weight_to_token_ratio=0.078850 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404830833e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44592 payload_rows_remote=38924 dup_rows=20944 dup_rows_remote=18242 dedup_ratio=0.319580 router_tokens_per_expert=[540, 521, 478, 490, 483, 511, 521, 482, 518, 542, 464, 489, 524, 461, 523, 508, 499, 487, 509, 536, 494, 503, 510, 506, 510, 533, 505, 530, 531, 523, 488, 530, 486, 533, 545, 508, 536, 536, 517, 524, 494, 530, 499, 521, 498, 498, 504, 525, 520, 495, 535, 522, 487, 515, 521, 481, 515, 541, 531, 522, 513, 546, 490, 531] payload_rows_per_dst=[5548, 5444, 5558, 5600, 5668, 5564, 5554, 5656] dup_rows_per_dst=[2504, 2614, 2530, 2700, 2702, 2574, 2598, 2722] combine_rows_per_src=[5548, 5444, 5558, 5600, 5668, 5564, 5554, 5656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44658 payload_rows_remote=38922 dup_rows=20878 dup_rows_remote=18110 dedup_ratio=0.318573 router_tokens_per_expert=[552, 549, 496, 480, 535, 504, 521, 489, 509, 492, 499, 513, 456, 505, 520, 493, 507, 538, 494, 492, 490, 502, 501, 525, 477, 497, 516, 482, 498, 505, 569, 538, 505, 515, 543, 488, 525, 497, 474, 526, 514, 532, 490, 499, 522, 519, 477, 493, 540, 519, 551, 510, 530, 543, 528, 531, 541, 511, 529, 544, 509, 495, 525, 499] payload_rows_per_dst=[5660, 5446, 5538, 5610, 5572, 5510, 5736, 5586] dup_rows_per_dst=[2592, 2528, 2560, 2554, 2574, 2582, 2768, 2720] combine_rows_per_src=[5660, 5446, 5538, 5610, 5572, 5510, 5736, 5586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44582 payload_rows_remote=38980 dup_rows=20954 dup_rows_remote=18200 dedup_ratio=0.319733 router_tokens_per_expert=[490, 481, 544, 519, 514, 518, 508, 505, 516, 506, 527, 496, 512, 487, 499, 495, 516, 520, 531, 542, 533, 513, 517, 484, 550, 512, 491, 515, 545, 519, 546, 522, 554, 504, 525, 488, 507, 508, 490, 514, 518, 488, 522, 479, 522, 511, 508, 502, 492, 488, 484, 514, 547, 491, 534, 479, 518, 513, 522, 509, 490, 535, 514, 525] payload_rows_per_dst=[5602, 5554, 5564, 5748, 5476, 5550, 5524, 5564] dup_rows_per_dst=[2754, 2536, 2536, 2982, 2478, 2664, 2534, 2470] combine_rows_per_src=[5602, 5554, 5564, 5748, 5476, 5550, 5524, 5564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159154176 dispatch_meta_bytes=228432 combine_payload_bytes=159154176 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=159382608 ingress_bytes=259817472 link_bytes=259817472 token_bytes=318536784 weight_movement_bytes=100663296 total_nvlink_bytes=419200080 weight_to_token_ratio=0.316018 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.976910759e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44416 payload_rows_remote=38906 dup_rows=21120 dup_rows_remote=18540 dedup_ratio=0.322266 router_tokens_per_expert=[540, 465, 450, 522, 549, 519, 513, 518, 501, 528, 545, 511, 546, 487, 535, 531, 536, 503, 535, 507, 492, 494, 508, 500, 496, 488, 498, 480, 469, 547, 537, 530, 532, 531, 557, 526, 514, 482, 514, 533, 507, 511, 482, 512, 499, 515, 524, 517, 492, 492, 509, 494, 550, 471, 484, 531, 522, 522, 497, 516, 539, 515, 464, 534] payload_rows_per_dst=[5522, 5604, 5508, 5510, 5656, 5570, 5494, 5552] dup_rows_per_dst=[2630, 2764, 2642, 2580, 2722, 2564, 2552, 2666] combine_rows_per_src=[5522, 5604, 5508, 5510, 5656, 5570, 5494, 5552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159432704 dispatch_meta_bytes=228664 combine_payload_bytes=159432704 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159661368 ingress_bytes=184598528 link_bytes=184598528 token_bytes=319094072 weight_movement_bytes=25165824 total_nvlink_bytes=344259896 weight_to_token_ratio=0.078866 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.404581506e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44650 payload_rows_remote=39034 dup_rows=20886 dup_rows_remote=18322 dedup_ratio=0.318695 router_tokens_per_expert=[526, 474, 529, 495, 494, 520, 498, 481, 529, 512, 532, 508, 483, 551, 494, 511, 516, 548, 501, 514, 521, 491, 525, 515, 515, 516, 543, 481, 500, 526, 526, 488, 494, 490, 497, 497, 479, 529, 522, 548, 519, 549, 505, 491, 493, 493, 498, 542, 488, 509, 519, 471, 546, 496, 552, 481, 509, 547, 517, 502, 527, 559, 485, 551] payload_rows_per_dst=[5506, 5568, 5524, 5602, 5552, 5616, 5596, 5686] dup_rows_per_dst=[2528, 2672, 2738, 2588, 2560, 2564, 2528, 2708] combine_rows_per_src=[5506, 5568, 5524, 5602, 5552, 5616, 5596, 5686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44466 payload_rows_remote=38838 dup_rows=21070 dup_rows_remote=18376 dedup_ratio=0.321503 router_tokens_per_expert=[542, 487, 512, 504, 500, 501, 515, 535, 490, 474, 507, 560, 545, 511, 517, 490, 533, 479, 508, 492, 548, 534, 545, 522, 536, 510, 538, 525, 519, 505, 524, 527, 498, 471, 509, 506, 555, 498, 493, 554, 500, 516, 510, 518, 516, 511, 506, 479, 503, 496, 487, 487, 552, 520, 507, 515, 487, 497, 512, 501, 501, 509, 511, 508] payload_rows_per_dst=[5560, 5554, 5628, 5702, 5540, 5518, 5502, 5462] dup_rows_per_dst=[2632, 2634, 2694, 2666, 2628, 2594, 2632, 2590] combine_rows_per_src=[5560, 5554, 5628, 5702, 5540, 5518, 5502, 5462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159424512 dispatch_meta_bytes=228128 combine_payload_bytes=159424512 prefetch_bytes=0 reduce_bytes=50331648 egress_bytes=159652640 ingress_bytes=209756160 link_bytes=209756160 token_bytes=319077152 weight_movement_bytes=50331648 total_nvlink_bytes=369408800 weight_to_token_ratio=0.157741 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.596002018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=159662080 dispatch_meta_bytes=228720 combine_payload_bytes=159662080 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159890800 ingress_bytes=184827904 link_bytes=184827904 token_bytes=319552880 weight_movement_bytes=25165824 total_nvlink_bytes=344718704 weight_to_token_ratio=0.078753 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.406326793e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44400 payload_rows_remote=38878 dup_rows=21136 dup_rows_remote=18454 dedup_ratio=0.322510 router_tokens_per_expert=[533, 515, 533, 488, 577, 495, 479, 539, 524, 526, 522, 543, 517, 494, 484, 492, 501, 531, 505, 531, 480, 463, 519, 506, 520, 532, 530, 575, 487, 511, 551, 524, 482, 511, 485, 498, 499, 496, 528, 520, 506, 558, 486, 502, 496, 533, 492, 509, 540, 536, 554, 537, 481, 525, 492, 504, 470, 509, 489, 512, 520, 481, 474, 516] payload_rows_per_dst=[5606, 5522, 5476, 5692, 5500, 5562, 5574, 5468] dup_rows_per_dst=[2712, 2682, 2596, 2768, 2538, 2602, 2764, 2474] combine_rows_per_src=[5606, 5522, 5476, 5692, 5500, 5562, 5574, 5468] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159358976 dispatch_meta_bytes=229784 combine_payload_bytes=159358976 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=159588760 ingress_bytes=234856448 link_bytes=234856448 token_bytes=318947736 weight_movement_bytes=75497472 total_nvlink_bytes=394445208 weight_to_token_ratio=0.236708 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.786986208e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159883264 dispatch_meta_bytes=229424 combine_payload_bytes=159883264 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160112688 ingress_bytes=185049088 link_bytes=185049088 token_bytes=319995952 weight_movement_bytes=25165824 total_nvlink_bytes=345161776 weight_to_token_ratio=0.078644 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408009747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=159080448 dispatch_meta_bytes=228856 combine_payload_bytes=159080448 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159309304 ingress_bytes=184246272 link_bytes=184246272 token_bytes=318389752 weight_movement_bytes=25165824 total_nvlink_bytes=343555576 weight_to_token_ratio=0.079041 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.401901245e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44310 payload_rows_remote=38760 dup_rows=21226 dup_rows_remote=18370 dedup_ratio=0.323883 router_tokens_per_expert=[530, 515, 518, 476, 489, 494, 505, 518, 489, 516, 556, 506, 498, 512, 513, 506, 515, 505, 513, 471, 508, 505, 526, 496, 530, 507, 512, 534, 507, 494, 508, 470, 522, 489, 527, 501, 497, 528, 492, 489, 488, 486, 485, 520, 538, 514, 526, 547, 515, 506, 543, 514, 553, 534, 500, 510, 535, 508, 507, 484, 553, 535, 527, 553] payload_rows_per_dst=[5526, 5476, 5468, 5416, 5570, 5644, 5660, 5550] dup_rows_per_dst=[2564, 2716, 2610, 2452, 2520, 2818, 2690, 2856] combine_rows_per_src=[5526, 5476, 5468, 5416, 5570, 5644, 5660, 5550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159244288 dispatch_meta_bytes=229328 combine_payload_bytes=159244288 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159473616 ingress_bytes=184410112 link_bytes=184410112 token_bytes=318717904 weight_movement_bytes=25165824 total_nvlink_bytes=343883728 weight_to_token_ratio=0.078960 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4352 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.403147878e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5109826000 prefetch_bytes=327155712 reduce_bytes=352321536 weight_movement_bytes=679477248 egress_bytes=2556748240 ingress_bytes=3232555008 link_bytes=3232555008 total_nvlink_bytes=5789303248 weight_to_token_ratio=0.132975 prefetch_share_of_ingress=0.101207 dedup_ratio=0.320614 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=14 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042050 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.537250537e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44616 payload_rows_remote=38988 dup_rows=20920 dup_rows_remote=18204 dedup_ratio=0.319214 router_tokens_per_expert=[515, 551, 482, 510, 517, 515, 512, 513, 503, 484, 553, 522, 491, 492, 510, 500, 530, 486, 494, 511, 505, 517, 500, 442, 514, 504, 530, 519, 499, 501, 503, 485, 504, 514, 530, 514, 566, 495, 528, 521, 568, 501, 524, 497, 473, 526, 510, 498, 467, 518, 513, 499, 486, 510, 571, 502, 529, 558, 547, 529, 533, 495, 523, 509] payload_rows_per_dst=[5604, 5526, 5428, 5528, 5628, 5570, 5600, 5732] dup_rows_per_dst=[2626, 2584, 2542, 2582, 2716, 2624, 2532, 2714] combine_rows_per_src=[5604, 5526, 5428, 5528, 5628, 5570, 5600, 5732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5106221528 prefetch_bytes=352321536 reduce_bytes=251658240 weight_movement_bytes=603979776 egress_bytes=2554946008 ingress_bytes=3155255296 link_bytes=3155255296 total_nvlink_bytes=5710201304 weight_to_token_ratio=0.118283 prefetch_share_of_ingress=0.111662 dedup_ratio=0.321386 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=10 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.500490444e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44510 payload_rows_remote=39012 dup_rows=21026 dup_rows_remote=18460 dedup_ratio=0.320831 router_tokens_per_expert=[540, 521, 531, 495, 527, 511, 521, 498, 518, 458, 525, 478, 546, 549, 498, 481, 504, 546, 513, 514, 549, 488, 498, 512, 513, 526, 519, 557, 509, 522, 542, 531, 485, 530, 547, 535, 523, 489, 514, 510, 528, 504, 518, 511, 508, 504, 466, 524, 509, 523, 487, 481, 516, 480, 492, 544, 510, 499, 520, 491, 498, 521, 475, 486] payload_rows_per_dst=[5654, 5462, 5578, 5674, 5604, 5542, 5498, 5498] dup_rows_per_dst=[2634, 2644, 2670, 2764, 2662, 2584, 2566, 2502] combine_rows_per_src=[5654, 5462, 5578, 5674, 5604, 5542, 5498, 5498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158760960 dispatch_meta_bytes=228520 combine_payload_bytes=158760960 prefetch_bytes=25165824 reduce_bytes=75497472 egress_bytes=158989480 ingress_bytes=259424256 link_bytes=259424256 token_bytes=317750440 weight_movement_bytes=100663296 total_nvlink_bytes=418413736 weight_to_token_ratio=0.316800 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.973918840e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5110742808 prefetch_bytes=377487360 reduce_bytes=150994944 weight_movement_bytes=528482304 egress_bytes=2557206296 ingress_bytes=3082018816 link_bytes=3082018816 total_nvlink_bytes=5639225112 weight_to_token_ratio=0.103406 prefetch_share_of_ingress=0.122481 dedup_ratio=0.320498 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=6 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.465662632e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44474 payload_rows_remote=38740 dup_rows=21062 dup_rows_remote=18102 dedup_ratio=0.321381 router_tokens_per_expert=[537, 534, 514, 523, 533, 505, 494, 534, 480, 457, 535, 527, 529, 499, 531, 514, 505, 494, 528, 484, 517, 508, 518, 488, 513, 506, 519, 509, 534, 521, 541, 485, 538, 507, 521, 525, 497, 531, 499, 541, 508, 501, 526, 521, 489, 509, 512, 506, 495, 507, 479, 524, 492, 530, 505, 473, 513, 549, 513, 499, 535, 483, 504, 520] payload_rows_per_dst=[5734, 5600, 5546, 5550, 5734, 5634, 5358, 5318] dup_rows_per_dst=[2960, 2638, 2620, 2632, 2726, 2770, 2372, 2344] combine_rows_per_src=[5734, 5600, 5546, 5550, 5734, 5634, 5358, 5318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159694848 dispatch_meta_bytes=228768 combine_payload_bytes=159694848 prefetch_bytes=25165824 reduce_bytes=25165824 egress_bytes=159923616 ingress_bytes=210026496 link_bytes=210026496 token_bytes=319618464 weight_movement_bytes=50331648 total_nvlink_bytes=369950112 weight_to_token_ratio=0.157474 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.598058962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159793152 dispatch_meta_bytes=229888 combine_payload_bytes=159793152 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160023040 ingress_bytes=184958976 link_bytes=184958976 token_bytes=319816192 weight_movement_bytes=25165824 total_nvlink_bytes=344982016 weight_to_token_ratio=0.078688 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.407324099e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5095580208 prefetch_bytes=327155712 reduce_bytes=629145600 weight_movement_bytes=956301312 egress_bytes=2549621296 ingress_bytes=3502260224 link_bytes=3502260224 total_nvlink_bytes=6051881520 weight_to_token_ratio=0.187673 prefetch_share_of_ingress=0.093413 dedup_ratio=0.322029 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=25 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041590 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.665509604e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44404 payload_rows_remote=38794 dup_rows=21132 dup_rows_remote=18466 dedup_ratio=0.322449 router_tokens_per_expert=[487, 524, 525, 523, 472, 516, 533, 501, 495, 492, 520, 518, 474, 524, 499, 544, 529, 523, 474, 555, 493, 533, 517, 514, 519, 547, 502, 501, 539, 558, 512, 506, 535, 511, 496, 537, 502, 533, 525, 526, 491, 504, 511, 493, 506, 476, 511, 498, 499, 524, 522, 522, 483, 509, 515, 506, 495, 498, 496, 529, 502, 540, 512, 492] payload_rows_per_dst=[5522, 5548, 5610, 5608, 5630, 5476, 5462, 5548] dup_rows_per_dst=[2640, 2584, 2666, 2760, 2700, 2504, 2698, 2580] combine_rows_per_src=[5522, 5548, 5610, 5608, 5630, 5476, 5462, 5548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=158679040 dispatch_meta_bytes=227368 combine_payload_bytes=158679040 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=158906408 ingress_bytes=183844864 link_bytes=183844864 token_bytes=317585448 weight_movement_bytes=25165824 total_nvlink_bytes=342751272 weight_to_token_ratio=0.079241 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.398846994e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5107348080 prefetch_bytes=352321536 reduce_bytes=251658240 weight_movement_bytes=603979776 egress_bytes=2555507312 ingress_bytes=3155820544 link_bytes=3155820544 total_nvlink_bytes=5711327856 weight_to_token_ratio=0.118257 prefetch_share_of_ingress=0.111642 dedup_ratio=0.320644 prefetch_slots=28 unused_prefetch_slots=228 slot_utilization=0.109375 reduce_slots=10 remote_demand=14 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.040901 gemm_inner_pad_upper_bound=20066 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.500759249e-04 prefetch_bytes_per_gemm_flop=1.675474877e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5109256160 prefetch_bytes=327155712 reduce_bytes=327155712 weight_movement_bytes=654311424 egress_bytes=2556465120 ingress_bytes=3207102464 link_bytes=3207332456 total_nvlink_bytes=5763567584 weight_to_token_ratio=0.128064 prefetch_share_of_ingress=0.102010 dedup_ratio=0.321049 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=13 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.525255883e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=158900224 dispatch_meta_bytes=229040 combine_payload_bytes=158900224 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=159129264 ingress_bytes=184066048 link_bytes=184066048 token_bytes=318029488 weight_movement_bytes=25165824 total_nvlink_bytes=343195312 weight_to_token_ratio=0.079130 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.400529949e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5103157008 prefetch_bytes=327155712 reduce_bytes=427819008 weight_movement_bytes=754974720 egress_bytes=2553413392 ingress_bytes=3304718336 link_bytes=3304718336 total_nvlink_bytes=5858131728 weight_to_token_ratio=0.147943 prefetch_share_of_ingress=0.098997 dedup_ratio=0.321243 prefetch_slots=26 unused_prefetch_slots=230 slot_utilization=0.101562 reduce_slots=17 remote_demand=13 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.041360 gemm_inner_pad_upper_bound=19939 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.571568009e-04 prefetch_bytes_per_gemm_flop=1.555798100e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=2048 assignments=32768 payload_rows=44574 payload_rows_remote=39036 dup_rows=20962 dup_rows_remote=18238 dedup_ratio=0.319855 router_tokens_per_expert=[517, 502, 491, 482, 519, 513, 546, 524, 508, 513, 535, 494, 510, 510, 486, 498, 505, 528, 522, 528, 522, 507, 514, 496, 539, 495, 489, 479, 482, 533, 516, 510, 518, 491, 573, 547, 496, 499, 489, 491, 536, 525, 491, 496, 546, 501, 525, 528, 509, 513, 501, 517, 540, 532, 511, 519, 509, 477, 538, 496, 501, 463, 526, 551] payload_rows_per_dst=[5538, 5580, 5684, 5520, 5456, 5584, 5688, 5524] dup_rows_per_dst=[2724, 2604, 2796, 2492, 2522, 2712, 2634, 2478] combine_rows_per_src=[5538, 5580, 5684, 5520, 5456, 5584, 5688, 5524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=12582912 grad_bytes_per_expert=25165824 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=159891456 dispatch_meta_bytes=229096 combine_payload_bytes=159891456 prefetch_bytes=25165824 reduce_bytes=0 egress_bytes=160120552 ingress_bytes=185057280 link_bytes=185057280 token_bytes=320012008 weight_movement_bytes=25165824 total_nvlink_bytes=345177832 weight_to_token_ratio=0.078640 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=4224 gemm_flops_fwd=438086664192 gemm_flops_step=1314259992576 bytes_per_gemm_flop=1.408072079e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=16 plans=16 dispatch_calls=32 combine_calls=32 prefetch_calls=32 reduce_calls=16 token_bytes=5101511536 prefetch_bytes=377487360 reduce_bytes=377487360 weight_movement_bytes=754974720 egress_bytes=2552587120 ingress_bytes=3303899136 link_bytes=3303899136 total_nvlink_bytes=5856486256 weight_to_token_ratio=0.147990 prefetch_share_of_ingress=0.114255 dedup_ratio=0.321041 prefetch_slots=30 unused_prefetch_slots=226 slot_utilization=0.117188 reduce_slots=15 remote_demand=15 gemm_rows_charged=557056 gemm_tail_pad_fraction=0.042509 gemm_inner_pad_upper_bound=20193 gemm_flops_step=21028159881216 bytes_per_gemm_flop=1.571178436e-04 prefetch_bytes_per_gemm_flop=1.795151654e-05 anomalies=none + [2026-09-21 08:32:51] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 216.3 | throughput per GPU (TFLOP/s/GPU): 149.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.640451E+00 | loss scale: 1.0 | grad norm: 3.488 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-09-21 08:32:51 +[rank0]:[W921 08:32:52.128307052 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 08:32:52.690000 677271 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 08:32:52.691000 677271 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 677449 closing signal SIGTERM +W0921 08:32:52.691000 677271 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 677450 closing signal SIGTERM +W0921 08:32:52.693000 677271 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 677451 closing signal SIGTERM +W0921 08:32:52.694000 677271 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 677452 closing signal SIGTERM +W0921 08:32:52.695000 677271 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 677453 closing signal SIGTERM +W0921 08:32:52.696000 677271 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 677454 closing signal SIGTERM +W0921 08:32:52.697000 677271 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 677455 closing signal SIGTERM +W0921 08:32:52.698000 677271 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 677456 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 677271 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d8dc73d9b796b9563060d904fcda74c0a76e3875 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -0,0 +1,3 @@ +model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir +OLMOE_1B_7B_L16 moonep natural none OK 35751 1 0 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1 +OLMOE_1B_7B_L16 moonep forced none OK 35751 1 0 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d1b7c08bb1d98ff910be096ff60addb703ff126c --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -0,0 +1,2 @@ +model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +OLMOE_1B_7B_L16 moonep SELECTED none 0 35751 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..545249d7b5f06915b3a187fc17a28b95be7ea187 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -0,0 +1,228 @@ +[ + { + "arm": "natural", + "aux_loss_coeff": "1e-2", + "backend": "moonep", + "branch": "moonep_backend_async_grad_reduce_token_routing_analysis", + "capacity": "none", + "capacity_mode": "none", + "cold_expert_fraction_max": 0.484375, + "cold_expert_fraction_mean": 0.2111328125, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 160, + "commit": "a6a1a221", + "config": "OLMOE_1B_7B_L16", + "cuda_graph": "0", + "dense_first_layers": "0", + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 160, + "hottest_expert_token_fraction_max": 0.99993896484375, + "hottest_expert_token_fraction_mean": 0.9486982345581054, + "hottest_expert_token_fraction_min": 0.390716552734375, + "hottest_expert_token_fraction_n": 160, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "moonep_bytes_global_snapshots": 160, + "moonep_bytes_incomplete_snapshots": 0, + "moonep_bytes_malformed_records": 0, + "moonep_global_combine_payload_bytes_max": 1867161600.0, + "moonep_global_combine_payload_bytes_mean": 1569235660.8, + "moonep_global_combine_payload_bytes_min": 1295032320.0, + "moonep_global_combine_payload_bytes_n": 160, + "moonep_global_dispatch_payload_bytes_max": 1867161600.0, + "moonep_global_dispatch_payload_bytes_mean": 1569235660.8, + "moonep_global_dispatch_payload_bytes_min": 1295032320.0, + "moonep_global_dispatch_payload_bytes_n": 160, + "moonep_global_prefetch_bytes_max": 251658240.0, + "moonep_global_prefetch_bytes_mean": 193934131.2, + "moonep_global_prefetch_bytes_min": 176160768.0, + "moonep_global_prefetch_bytes_n": 160, + "moonep_global_reduce_bytes_max": 251658240.0, + "moonep_global_reduce_bytes_mean": 193934131.2, + "moonep_global_reduce_bytes_min": 176160768.0, + "moonep_global_reduce_bytes_n": 160, + "moonep_global_token_bytes_max": 3736155704.0, + "moonep_global_token_bytes_mean": 3140265240.85, + "moonep_global_token_bytes_min": 2591880616.0, + "moonep_global_token_bytes_n": 160, + "moonep_global_total_nvlink_bytes_max": 4138808888.0, + "moonep_global_total_nvlink_bytes_mean": 3528133503.25, + "moonep_global_total_nvlink_bytes_min": 2944202152.0, + "moonep_global_total_nvlink_bytes_n": 160, + "moonep_global_weight_movement_bytes_max": 503316480.0, + "moonep_global_weight_movement_bytes_mean": 387868262.4, + "moonep_global_weight_movement_bytes_min": 352321536.0, + "moonep_global_weight_movement_bytes_n": 160, + "moonep_global_weight_to_token_ratio_max": 0.1934270379594032, + "moonep_global_weight_to_token_ratio_mean": 0.12484131355395116, + "moonep_global_weight_to_token_ratio_min": 0.09462042088977833, + "moonep_global_weight_to_token_ratio_n": 160, + "moonep_remote_demand_max": 3.0, + "moonep_remote_demand_mean": 0.96328125, + "moonep_remote_demand_min": 0.0, + "moonep_remote_demand_n": 1280, + "moonep_slot_utilization_max": 0.375, + "moonep_slot_utilization_mean": 0.12041015625, + "moonep_slot_utilization_min": 0.0, + "moonep_slot_utilization_n": 1280, + "moonep_unused_prefetch_slots_max": 16.0, + "moonep_unused_prefetch_slots_mean": 14.0734375, + "moonep_unused_prefetch_slots_min": 10.0, + "moonep_unused_prefetch_slots_n": 1280, + "num_experts": "64", + "num_layers": "16", + "post_physical_rank_cv_max": 0.22561764409106666, + "post_physical_rank_cv_mean": 0.09623031930856718, + "post_physical_rank_cv_min": 0.004191678422331892, + "post_physical_rank_cv_n": 160, + "post_physical_rank_imax_max": 1.3098809189892138, + "post_physical_rank_imax_mean": 1.1295311977570006, + "post_physical_rank_imax_min": 1.0052335128996117, + "post_physical_rank_imax_n": 160, + "pre_home_rank_cv_max": 1.304032801068936, + "pre_home_rank_cv_mean": 0.7290410654343018, + "pre_home_rank_cv_min": 0.23067981190173764, + "pre_home_rank_cv_n": 160, + "pre_home_rank_imax_max": 3.941009521484375, + "pre_home_rank_imax_mean": 2.289567756652832, + "pre_home_rank_imax_min": 1.311187744140625, + "pre_home_rank_imax_n": 160, + "raw_expert_cv_max": 2.6256940664188297, + "raw_expert_cv_mean": 2.1930980791488244, + "raw_expert_cv_min": 0.8856034659168336, + "raw_expert_cv_n": 160, + "raw_expert_imax_ceiling": 8.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 7.99951171875, + "raw_expert_imax_mean": 7.589585876464843, + "raw_expert_imax_min": 3.125732421875, + "raw_expert_imax_n": 160, + "records": { + "moonep_bytes": 1280, + "moonep_routing": 1280, + "moonep_summary": 80, + "moonep_weights": 1280 + }, + "router_balancing": "aux_loss", + "run_dir": "OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8" + }, + { + "arm": "forced", + "aux_loss_coeff": "1e-2", + "backend": "moonep", + "branch": "moonep_backend_async_grad_reduce_token_routing_analysis", + "capacity": "none", + "capacity_mode": "none", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 160, + "commit": "a6a1a221", + "config": "OLMOE_1B_7B_L16", + "cuda_graph": "0", + "dense_first_layers": "0", + "experts_per_rank": "8", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 160, + "hottest_expert_token_fraction_max": 0.13189697265625, + "hottest_expert_token_fraction_mean": 0.1294095993041992, + "hottest_expert_token_fraction_min": 0.1278076171875, + "hottest_expert_token_fraction_n": 160, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "moonep_bytes_global_snapshots": 160, + "moonep_bytes_incomplete_snapshots": 0, + "moonep_bytes_malformed_records": 0, + "moonep_global_combine_payload_bytes_max": 1278230528.0, + "moonep_global_combine_payload_bytes_mean": 1275283558.4, + "moonep_global_combine_payload_bytes_min": 1271652352.0, + "moonep_global_combine_payload_bytes_n": 160, + "moonep_global_dispatch_payload_bytes_max": 1278230528.0, + "moonep_global_dispatch_payload_bytes_mean": 1275283558.4, + "moonep_global_dispatch_payload_bytes_min": 1271652352.0, + "moonep_global_dispatch_payload_bytes_n": 160, + "moonep_global_prefetch_bytes_max": 176160768.0, + "moonep_global_prefetch_bytes_mean": 173486899.2, + "moonep_global_prefetch_bytes_min": 125829120.0, + "moonep_global_prefetch_bytes_n": 160, + "moonep_global_reduce_bytes_max": 176160768.0, + "moonep_global_reduce_bytes_mean": 173486899.2, + "moonep_global_reduce_bytes_min": 125829120.0, + "moonep_global_reduce_bytes_n": 160, + "moonep_global_token_bytes_max": 2558296472.0, + "moonep_global_token_bytes_mean": 2552401001.25, + "moonep_global_token_bytes_min": 2545135352.0, + "moonep_global_token_bytes_n": 160, + "moonep_global_total_nvlink_bytes_max": 2910618008.0, + "moonep_global_total_nvlink_bytes_mean": 2899374799.65, + "moonep_global_total_nvlink_bytes_min": 2805284064.0, + "moonep_global_total_nvlink_bytes_n": 160, + "moonep_global_weight_movement_bytes_max": 352321536.0, + "moonep_global_weight_movement_bytes_mean": 346973798.4, + "moonep_global_weight_movement_bytes_min": 251658240.0, + "moonep_global_weight_movement_bytes_n": 160, + "moonep_global_weight_to_token_ratio_max": 0.1384293906896312, + "moonep_global_weight_to_token_ratio_mean": 0.1359406755873575, + "moonep_global_weight_to_token_ratio_min": 0.09854937933146465, + "moonep_global_weight_to_token_ratio_n": 160, + "moonep_remote_demand_max": 1.0, + "moonep_remote_demand_mean": 0.86171875, + "moonep_remote_demand_min": 0.0, + "moonep_remote_demand_n": 1280, + "moonep_slot_utilization_max": 0.125, + "moonep_slot_utilization_mean": 0.10771484375, + "moonep_slot_utilization_min": 0.0, + "moonep_slot_utilization_n": 1280, + "moonep_unused_prefetch_slots_max": 16.0, + "moonep_unused_prefetch_slots_mean": 14.2765625, + "moonep_unused_prefetch_slots_min": 14.0, + "moonep_unused_prefetch_slots_n": 1280, + "num_experts": "64", + "num_layers": "16", + "post_physical_rank_cv_max": 0.004151405986167901, + "post_physical_rank_cv_mean": 0.0021700897212804954, + "post_physical_rank_cv_min": 0.0007203705850357968, + "post_physical_rank_cv_n": 160, + "post_physical_rank_imax_max": 1.0083050780744813, + "post_physical_rank_imax_mean": 1.0033887240372406, + "post_physical_rank_imax_min": 1.0013429305103698, + "post_physical_rank_imax_n": 160, + "pre_home_rank_cv_max": 0.009250589744190636, + "pre_home_rank_cv_mean": 0.004649780487546198, + "pre_home_rank_cv_min": 0.001428470427059137, + "pre_home_rank_cv_n": 160, + "pre_home_rank_imax_max": 1.015838623046875, + "pre_home_rank_imax_mean": 1.007100486755371, + "pre_home_rank_imax_min": 1.002044677734375, + "pre_home_rank_imax_n": 160, + "raw_expert_cv_max": 0.017802554164692116, + "raw_expert_cv_mean": 0.014583830753424936, + "raw_expert_cv_min": 0.010177326341354839, + "raw_expert_cv_n": 160, + "raw_expert_imax_ceiling": 8.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.05517578125, + "raw_expert_imax_mean": 1.0352767944335937, + "raw_expert_imax_min": 1.0224609375, + "raw_expert_imax_n": 160, + "records": { + "moonep_bytes": 1280, + "moonep_routing": 1280, + "moonep_summary": 80, + "moonep_weights": 1280 + }, + "router_balancing": "aux_loss", + "run_dir": "OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8" + } +] \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..96946a7880f5196c04cf02f73d6f5f4786495f37 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -0,0 +1,3 @@ +run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records moonep_bytes_global_snapshots moonep_bytes_incomplete_snapshots moonep_bytes_malformed_records moonep_global_combine_payload_bytes_max moonep_global_combine_payload_bytes_mean moonep_global_combine_payload_bytes_min moonep_global_combine_payload_bytes_n moonep_global_dispatch_payload_bytes_max moonep_global_dispatch_payload_bytes_mean moonep_global_dispatch_payload_bytes_min moonep_global_dispatch_payload_bytes_n moonep_global_prefetch_bytes_max moonep_global_prefetch_bytes_mean moonep_global_prefetch_bytes_min moonep_global_prefetch_bytes_n moonep_global_reduce_bytes_max moonep_global_reduce_bytes_mean moonep_global_reduce_bytes_min moonep_global_reduce_bytes_n moonep_global_token_bytes_max moonep_global_token_bytes_mean moonep_global_token_bytes_min moonep_global_token_bytes_n moonep_global_total_nvlink_bytes_max moonep_global_total_nvlink_bytes_mean moonep_global_total_nvlink_bytes_min moonep_global_total_nvlink_bytes_n moonep_global_weight_movement_bytes_max moonep_global_weight_movement_bytes_mean moonep_global_weight_movement_bytes_min moonep_global_weight_movement_bytes_n moonep_global_weight_to_token_ratio_max moonep_global_weight_to_token_ratio_mean moonep_global_weight_to_token_ratio_min moonep_global_weight_to_token_ratio_n moonep_remote_demand_max moonep_remote_demand_mean moonep_remote_demand_min moonep_remote_demand_n moonep_slot_utilization_max moonep_slot_utilization_mean moonep_slot_utilization_min moonep_slot_utilization_n moonep_unused_prefetch_slots_max moonep_unused_prefetch_slots_mean moonep_unused_prefetch_slots_min moonep_unused_prefetch_slots_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n +OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1 natural OLMOE_1B_7B_L16 moonep 64 8 8 16 16 0 4096 8 none none 0 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 aux_loss 1e-2 0 0.484375 0.2111328125 0.0 160 160 0.99993896484375 0.9486982345581054 0.390716552734375 160 0 0 160 0 0 1867161600.0 1569235660.8 1295032320.0 160 1867161600.0 1569235660.8 1295032320.0 160 251658240.0 193934131.2 176160768.0 160 251658240.0 193934131.2 176160768.0 160 3736155704.0 3140265240.85 2591880616.0 160 4138808888.0 3528133503.25 2944202152.0 160 503316480.0 387868262.4 352321536.0 160 0.1934270379594032 0.12484131355395116 0.09462042088977833 160 3.0 0.96328125 0.0 1280 0.375 0.12041015625 0.0 1280 16.0 14.0734375 10.0 1280 0.22561764409106666 0.09623031930856718 0.004191678422331892 160 1.3098809189892138 1.1295311977570006 1.0052335128996117 160 1.304032801068936 0.7290410654343018 0.23067981190173764 160 3.941009521484375 2.289567756652832 1.311187744140625 160 2.6256940664188297 2.1930980791488244 0.8856034659168336 160 8.0 1 7.99951171875 7.589585876464843 3.125732421875 160 +OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1 forced OLMOE_1B_7B_L16 moonep 64 8 8 16 16 0 4096 8 none none 1 0 moonep_backend_async_grad_reduce_token_routing_analysis a6a1a221 aux_loss 1e-2 0 0.0 0.0 0.0 160 160 0.13189697265625 0.1294095993041992 0.1278076171875 160 0 0 160 0 0 1278230528.0 1275283558.4 1271652352.0 160 1278230528.0 1275283558.4 1271652352.0 160 176160768.0 173486899.2 125829120.0 160 176160768.0 173486899.2 125829120.0 160 2558296472.0 2552401001.25 2545135352.0 160 2910618008.0 2899374799.65 2805284064.0 160 352321536.0 346973798.4 251658240.0 160 0.1384293906896312 0.1359406755873575 0.09854937933146465 160 1.0 0.86171875 0.0 1280 0.125 0.10771484375 0.0 1280 16.0 14.2765625 14.0 1280 0.004151405986167901 0.0021700897212804954 0.0007203705850357968 160 1.0083050780744813 1.0033887240372406 1.0013429305103698 160 0.009250589744190636 0.004649780487546198 0.001428470427059137 160 1.015838623046875 1.007100486755371 1.002044677734375 160 0.017802554164692116 0.014583830753424936 0.010177326341354839 160 8.0 0 1.05517578125 1.0352767944335937 1.0224609375 160 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/results.tsv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..de6cde9b6c630670162c36187711c90d8c0b85d7 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -0,0 +1,3 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +OLMOE_1B_7B_L16 moonep matched 8 4096 8 none none 1 OK 220.81 215.30 35751 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__async/rep1 +OLMOE_1B_7B_L16 moonep matched 8 4096 8 none none+forcebal 1 OK 212.30 210.50 35751 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/moonep/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__moonep__matched__seq4096__gbs8__capnone__forcebal__async/rep1